You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/02/05 22:54:03 UTC
[1/4] incubator-usergrid git commit: adding retry count.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-import 0bf0d81de -> 7d27c230f
adding retry count.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7ec1e055
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7ec1e055
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7ec1e055
Branch: refs/heads/two-dot-o-import
Commit: 7ec1e0551bc8c341a95b794a147b2e3bc723457e
Parents: fd4d47b
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Feb 5 12:19:13 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Feb 5 12:19:13 2015 -0500
----------------------------------------------------------------------
.../usergrid/management/importer/ImportCollectionIT.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7ec1e055/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index 168cec7..a430961 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -380,8 +380,9 @@ public class ImportCollectionIT {
// listener.start();
- // TODO countdown latch here?
- while ( !importService.getState( importUUID ).equals( "FINISHED" ) ) {
+ int maxRetries = 0;
+ int retries = 100;
+ while ( !importService.getState( importUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
Thread.sleep(100);
}
@@ -418,8 +419,9 @@ public class ImportCollectionIT {
}});
}});
- // TODO countdown latch here?
- while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {
+ int maxRetries = 0;
+ int retries = 100;
+ while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
Thread.sleep(100);
}
}
[2/4] incubator-usergrid git commit: Merge branch 'two-dot-o-import'
of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-import
Posted by sn...@apache.org.
Merge branch 'two-dot-o-import' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-import
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b3c3e94f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b3c3e94f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b3c3e94f
Branch: refs/heads/two-dot-o-import
Commit: b3c3e94ffce16a5993d15984eee233d9c901cd40
Parents: 7ec1e05 0bf0d81
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Feb 5 12:19:25 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Feb 5 12:19:25 2015 -0500
----------------------------------------------------------------------
.../management/importer/ImportCollectionIT.java | 30 ++++++++++++++------
1 file changed, 21 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b3c3e94f/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
[4/4] incubator-usergrid git commit: Progress but still have one
ExportServiceIT test and one ImportCollectionIT test failing.
Posted by sn...@apache.org.
Progress but still have one ExportServiceIT test and one ImportCollectionIT test failing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7d27c230
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7d27c230
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7d27c230
Branch: refs/heads/two-dot-o-import
Commit: 7d27c230f1838f0990b125c0a1b270ea38c72aec
Parents: be1b9ff
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Feb 5 16:47:24 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Feb 5 16:47:24 2015 -0500
----------------------------------------------------------------------
.../management/export/ExportServiceIT.java | 84 +++---
.../management/importer/ImportCollectionIT.java | 284 +++++++++++--------
2 files changed, 219 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7d27c230/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
index 7cb41e7..9d724c1 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
@@ -87,18 +87,17 @@ public class ExportServiceIT {
@Rule
public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
-
@Rule
- public NewOrgAppAdminRule orgAppAdminRule = new NewOrgAppAdminRule( setup );
-
+ public NewOrgAppAdminRule newOrgAppAdminRule = new NewOrgAppAdminRule( setup );
// app-level data generated only once
private UserInfo adminUser;
private OrganizationInfo organization;
private UUID applicationId;
- final String bucketName = System.getProperty( "bucketName" )
- + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
+ private static String bucketPrefix;
+
+ private String bucketName;
@Before
public void setup() throws Exception {
@@ -110,9 +109,9 @@ public class ExportServiceIT {
jobScheduler.startAndWait();
}
- adminUser = orgAppAdminRule.getAdminInfo();
- organization = orgAppAdminRule.getOrganizationInfo();
- applicationId = orgAppAdminRule.getApplicationInfo().getId();
+ adminUser = newOrgAppAdminRule.getAdminInfo();
+ organization = newOrgAppAdminRule.getOrganizationInfo();
+ applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
setup.getEmf().refreshIndex();
}
@@ -122,16 +121,27 @@ public class ExportServiceIT {
public void before() {
boolean configured =
- !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ))
- && !StringUtils.isEmpty(System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ))
- && !StringUtils.isEmpty(System.getProperty("bucketName"));
+ !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR))
+ && !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR))
+ && !StringUtils.isEmpty(System.getProperty("bucketName"));
if ( !configured ) {
- logger.warn("Skipping test because accessKey, secretKey and bucketName not " +
- "specified as system properties, e.g. in your Maven settings.xml file.");
+ logger.warn("Skipping test because {}, {} and bucketName not " +
+ "specified as system properties, e.g. in your Maven settings.xml file.",
+ new Object[] {
+ SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
+ SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR
+ });
}
- Assume.assumeTrue(configured);
+ Assume.assumeTrue( configured );
+
+ adminUser = newOrgAppAdminRule.getAdminInfo();
+ organization = newOrgAppAdminRule.getOrganizationInfo();
+ applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
+
+ bucketPrefix = System.getProperty( "bucketName" );
+ bucketName = bucketPrefix + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
}
@@ -155,7 +165,7 @@ public class ExportServiceIT {
ExportService exportService = setup.getExportService();
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
@@ -231,18 +241,20 @@ public class ExportServiceIT {
ExportService exportService = setup.getExportService();
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
EntityManager em = setup.getEmf().getEntityManager( applicationId );
- //intialize user object to be posted
+
+ // intialize user object to be posted
Map<String, Object> userProperties = null;
Entity[] entity;
entity = new Entity[2];
- //creates entities
+
+ // creates entities
for ( int i = 0; i < 2; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
@@ -313,7 +325,7 @@ public class ExportServiceIT {
}
//create another org to ensure we don't export it
- orgAppAdminRule.createOwnerAndOrganization(
+ newOrgAppAdminRule.createOwnerAndOrganization(
"noExport"+newUUIDString(),
"junkUserName"+newUUIDString(),
"junkRealName"+newUUIDString(),
@@ -324,7 +336,7 @@ public class ExportServiceIT {
// s3Export.setFilename( "exportOneOrg.json" );
ExportService exportService = setup.getExportService();
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
@@ -457,7 +469,7 @@ public class ExportServiceIT {
S3Export s3Export = new MockS3ExportImpl("exportOneAppWQuery.json" );
ExportService exportService = setup.getExportService();
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "query", "select * where username = 'junkRealName'" );
@@ -505,8 +517,8 @@ public class ExportServiceIT {
f.deleteOnExit();
EntityManager em = setup.getEmf().getEntityManager( applicationId );
- // em.createApplicationCollection( "qtsMagics" );
- //intialize user object to be posted
+ // em.createApplicationCollection( "qtsMagics" );
+ // intialize user object to be posted
Map<String, Object> userProperties = null;
Entity[] entity;
entity = new Entity[entitiesToCreate];
@@ -521,7 +533,7 @@ public class ExportServiceIT {
S3Export s3Export = new MockS3ExportImpl("exportOneCollection.json" );
ExportService exportService = setup.getExportService();
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
@@ -566,12 +578,12 @@ public class ExportServiceIT {
em.createApplicationCollection( "baconators" );
em.refreshIndex();
- //intialize user object to be posted
+ //initialize user object to be posted
Map<String, Object> userProperties = null;
Entity[] entity;
entity = new Entity[entitiesToCreate];
- //creates entities
+ // creates entities
for ( int i = 0; i < entitiesToCreate; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
@@ -582,7 +594,7 @@ public class ExportServiceIT {
S3Export s3Export = new MockS3ExportImpl("exportOneCollectionWQuery.json");
ExportService exportService = setup.getExportService();
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "query", "select * where username contains 'billybob0'" );
@@ -605,7 +617,7 @@ public class ExportServiceIT {
org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- //only one entity should match the query.
+ // only one entity should match the query.
assertEquals( 1, a.size() );
}
@@ -662,7 +674,7 @@ public class ExportServiceIT {
JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
JobExecution jobExecution = mock( JobExecution.class );
- when( jobExecution.getJobData() ).thenReturn( jobData );
+ when( jobExecution.getJobData() ).thenReturn(jobData);
exportService.doExport( jobExecution );
@@ -675,14 +687,14 @@ public class ExportServiceIT {
org.json.simple.JSONArray a = ( org.json.simple.JSONArray )
parser.parse( new FileReader( exportedFile ) );
- assertEquals( 23, a.size() );
+ assertEquals(23, a.size());
}
@Test
public void testExportDoJob() throws Exception {
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
@@ -769,7 +781,7 @@ public class ExportServiceIT {
ExportService exportService = setup.getExportService();
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
@@ -801,8 +813,8 @@ public class ExportServiceIT {
UUID exportUUID = exportService.schedule( payload );
- int maxRetries = 0;
- int retries = 100;
+ int maxRetries = 100;
+ int retries = 0;
while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
Thread.sleep(100);
}
@@ -857,7 +869,7 @@ public class ExportServiceIT {
S3Export s3Export = new S3ExportImpl();
ExportService exportService = setup.getExportService();
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
OrganizationInfo orgMade = null;
@@ -940,7 +952,7 @@ public class ExportServiceIT {
S3Export s3Export = new S3ExportImpl();
ExportService exportService = setup.getExportService();
- String appName = orgAppAdminRule.getApplicationInfo().getName();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7d27c230/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index 89101e5..676bdac 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -30,17 +30,20 @@ import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.batch.service.JobSchedulerService;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.cassandra.ClearShiroSubject;
+import org.apache.usergrid.cassandra.Concurrent;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.management.export.ExportService;
import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.persistence.index.query.Query.Level;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.services.notifications.QueueListener;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.ContainerNotFoundException;
+import org.jclouds.blobstore.domain.PageSet;
+import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.netty.config.NettyPayloadModule;
@@ -54,12 +57,11 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-//@Concurrent These tests cannot be run concurrently
+@Concurrent
public class ImportCollectionIT {
private static final Logger logger = LoggerFactory.getLogger(ImportCollectionIT.class);
-
private static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();
// app-level data generated only once
@@ -67,9 +69,9 @@ public class ImportCollectionIT {
private static OrganizationInfo organization;
private static UUID applicationId;
- private String bucketName;
+ private static String bucketPrefix;
- QueueListener listener;
+ private String bucketName;
@Rule
@@ -79,26 +81,27 @@ public class ImportCollectionIT {
public static final ServiceITSetup setup =
new ServiceITSetupImpl( cassandraResource, new ElasticSearchResource() );
-
@Rule
public NewOrgAppAdminRule newOrgAppAdminRule = new NewOrgAppAdminRule( setup );
@BeforeClass
public static void setup() throws Exception {
-// String username = "test"+ UUIDUtils.newTimeUUID();
// start the scheduler after we're all set up
JobSchedulerService jobScheduler = cassandraResource.getBean( JobSchedulerService.class );
if ( jobScheduler.state() != Service.State.RUNNING ) {
jobScheduler.startAndWait();
}
-//
-// //creates sample test application
-// adminUser = setup.getMgmtSvc().createAdminUser(
-// username, username, username+"@test.com", username, false, false );
-// organization = setup.getMgmtSvc().createOrganization( username, adminUser, true );
-// applicationId = setup.getMgmtSvc().createApplication( organization.getUuid(), username+"app" ).getId();
+
+ }
+
+
+ @AfterClass
+ public static void tearDown() {
+// if ( !StringUtils.isEmpty( bucketPrefix )) {
+// deleteBucketsWithPrefix();
+// }
}
@@ -121,20 +124,21 @@ public class ImportCollectionIT {
Assume.assumeTrue( configured );
-
adminUser = newOrgAppAdminRule.getAdminInfo();
organization = newOrgAppAdminRule.getOrganizationInfo();
applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
- bucketName = System.getProperty( "bucketName" )+ RandomStringUtils.randomAlphanumeric(10).toLowerCase();
+
+ bucketPrefix = System.getProperty( "bucketName" );
+ bucketName = bucketPrefix + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
}
@After
public void after() throws Exception {
- if(listener != null) {
- listener.stop();
- listener = null;
- }
+// if (listener != null) {
+// listener.stop();
+// listener = null;
+// }
}
@@ -143,27 +147,27 @@ public class ImportCollectionIT {
public void testExportImportCollection() throws Exception {
// create a collection of "thing" entities in the first application, export to S3
+ try {
- final EntityManager emApp1 = setup.getEmf().getEntityManager( applicationId );
- Map<UUID, Entity> thingsMap = new HashMap<>();
- List<Entity> things = new ArrayList<>();
- createTestEntities(emApp1, thingsMap, things, "thing");
+ final EntityManager emApp1 = setup.getEmf().getEntityManager( applicationId );
+ Map<UUID, Entity> thingsMap = new HashMap<>();
+ List<Entity> things = new ArrayList<>();
+ createTestEntities(emApp1, thingsMap, things, "thing");
- deleteBucket();
- exportCollection( emApp1, "things" );
+ deleteBucket();
+ exportCollection( emApp1, "things" );
- // create new second application, import the data from S3
+ // create new second application, import the data from S3
- final UUID appId2 = setup.getMgmtSvc().createApplication(
- organization.getUuid(), "second").getId();
+ final UUID appId2 = setup.getMgmtSvc().createApplication(
+ organization.getUuid(), "second").getId();
- final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
- importCollection( emApp2, "things" );
+ final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
+ importCollection( emApp2, "things" );
- // make sure that it worked
+ // make sure that it worked
- try {
logger.debug("\n\nCheck connections\n");
List<Entity> importedThings = emApp2.getCollection(
@@ -217,8 +221,7 @@ public class ImportCollectionIT {
appId2, "things", null, Level.ALL_PROPERTIES).getEntities();
assertTrue( !importedThings.isEmpty() );
- }
- finally {
+ } finally {
deleteBucket();
}
}
@@ -241,73 +244,84 @@ public class ImportCollectionIT {
createTestEntities(emApp1, thingsMap, things, "thing");
deleteBucket();
- exportCollection( emApp1, "things" );
+ try {
+ exportCollection(emApp1, "things");
- // create new second application and import those things from S3
+ // create new second application and import those things from S3
- final UUID appId2 = setup.getMgmtSvc().createApplication(
- organization.getUuid(), "second").getId();
+ final UUID appId2 = setup.getMgmtSvc().createApplication(
+ organization.getUuid(), "second").getId();
- final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
- importCollection( emApp2, "things" );
+ final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
+ importCollection(emApp2, "things");
- // update the things in the second application, export to S3
+ // update the things in the second application, export to S3
- for ( UUID uuid : thingsMap.keySet() ) {
- Entity entity = emApp2.get( uuid );
- entity.setProperty("fuel_source", "Hydrogen");
- emApp2.update( entity );
- }
+ for (UUID uuid : thingsMap.keySet()) {
+ Entity entity = emApp2.get(uuid);
+ entity.setProperty("fuel_source", "Hydrogen");
+ emApp2.update(entity);
+ }
- deleteBucket();
- exportCollection( emApp2, "things" );
+ deleteBucket();
+ exportCollection(emApp2, "things");
- // import the updated things back into the first application, check that they've been updated
+ // import the updated things back into the first application, check that they've been updated
- importCollection(emApp1, "things");
+ importCollection(emApp1, "things");
- for ( UUID uuid : thingsMap.keySet() ) {
- Entity entity = emApp1.get( uuid );
- Assert.assertEquals("Hydrogen", entity.getProperty("fuel_source"));
+ for (UUID uuid : thingsMap.keySet()) {
+ Entity entity = emApp1.get(uuid);
+ Assert.assertEquals("Hydrogen", entity.getProperty("fuel_source"));
+ }
+
+ } finally {
+ deleteBucket();
}
}
/**
- * Test that the types of incoming entities is ignored.
+ * Test that the types of i* ncoming entities is ignored.
*/
@Test
public void testImportWithWrongTypes() throws Exception {
deleteBucket();
- // create an app with a collection of cats, export it to S3
+ try {
+
+ // create an app with a collection of cats, export it to S3
- String appName = "import-test-" + RandomStringUtils.randomAlphanumeric(10);
- UUID appId = setup.getMgmtSvc().createApplication(organization.getUuid(), appName).getId();
+ String appName = "import-test-" + RandomStringUtils.randomAlphanumeric(10);
+ UUID appId = setup.getMgmtSvc().createApplication(organization.getUuid(), appName).getId();
- Map<UUID, Entity> catsMap = new HashMap<>();
- List<Entity> cats = new ArrayList<>();
+ Map<UUID, Entity> catsMap = new HashMap<>();
+ List<Entity> cats = new ArrayList<>();
- EntityManager emApp = setup.getEmf().getEntityManager( appId );
- createTestEntities( emApp, catsMap, cats, "cat");
- exportCollection(emApp, "cats");
+ EntityManager emApp = setup.getEmf().getEntityManager(appId);
+ createTestEntities(emApp, catsMap, cats, "cat");
+ exportCollection(emApp, "cats");
- // import the cats data into a new collection called dogs in the default test app
+ // import the cats data into a new collection called dogs in the default test app
- final EntityManager emDefaultApp = setup.getEmf().getEntityManager( applicationId );
- importCollection( emDefaultApp, "dogs" );
+ final EntityManager emDefaultApp = setup.getEmf().getEntityManager(applicationId);
+ importCollection(emDefaultApp, "dogs");
- // check that we now have a collection of dogs in the default test app
+ // check that we now have a collection of dogs in the default test app
- List<Entity> importedThings = emDefaultApp.getCollection(
- emDefaultApp.getApplicationId(), "dogs", null, Level.ALL_PROPERTIES).getEntities();
+ List<Entity> importedThings = emDefaultApp.getCollection(
+ emDefaultApp.getApplicationId(), "dogs", null, Level.ALL_PROPERTIES).getEntities();
- assertTrue( !importedThings.isEmpty() );
- assertEquals( 10, importedThings.size() );
+ assertTrue(!importedThings.isEmpty());
+ assertEquals(10, importedThings.size());
+
+ } finally {
+ deleteBucket();
+ }
}
@@ -320,31 +334,37 @@ public class ImportCollectionIT {
deleteBucket();
- // create 10 applications each with collection of 10 things, export all to S3
+ try {
- Map<UUID, Entity> thingsMap = new HashMap<>();
- List<Entity> things = new ArrayList<>();
+ // create 10 applications each with collection of 10 things, export all to S3
- for ( int i=0; i<10; i++) {
- String appName = "import-test-" + i + RandomStringUtils.randomAlphanumeric(10);
- UUID appId = setup.getMgmtSvc().createApplication(organization.getUuid(), appName).getId();
- EntityManager emApp = setup.getEmf().getEntityManager( appId );
- createTestEntities( emApp, thingsMap, things, "thing" );
- exportCollection( emApp, "things" );
- }
+ Map<UUID, Entity> thingsMap = new HashMap<>();
+ List<Entity> things = new ArrayList<>();
+
+ for (int i = 0; i < 10; i++) {
+ String appName = "import-test-" + i + RandomStringUtils.randomAlphanumeric(10);
+ UUID appId = setup.getMgmtSvc().createApplication(organization.getUuid(), appName).getId();
+ EntityManager emApp = setup.getEmf().getEntityManager(appId);
+ createTestEntities(emApp, thingsMap, things, "thing");
+ exportCollection(emApp, "things");
+ }
- // import all those exports from S3 into the default test application
+ // import all those exports from S3 into the default test application
- final EntityManager emDefaultApp = setup.getEmf().getEntityManager( applicationId );
- importCollection( emDefaultApp, "things" );
+ final EntityManager emDefaultApp = setup.getEmf().getEntityManager(applicationId);
+ importCollection(emDefaultApp, "things");
- // we should now have 100 Entities in the default app
+ // we should now have 100 Entities in the default app
- List<Entity> importedThings = emDefaultApp.getCollection(
- emDefaultApp.getApplicationId(), "things", null, Level.ALL_PROPERTIES).getEntities();
+ List<Entity> importedThings = emDefaultApp.getCollection(
+ emDefaultApp.getApplicationId(), "things", null, Level.ALL_PROPERTIES).getEntities();
- assertTrue( !importedThings.isEmpty() );
- assertEquals( 100, importedThings.size() );
+ assertTrue(!importedThings.isEmpty());
+ assertEquals(100, importedThings.size());
+
+ } finally {
+ deleteBucket();
+ }
}
@@ -385,15 +405,15 @@ public class ImportCollectionIT {
System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ) );
- put( "bucket_location", bucketName);
+ put( "bucket_location", bucketName );
}});
}});
}});
// listener.start();
- int maxRetries = 0;
- int retries = 100;
+ int maxRetries = 100;
+ int retries = 0;
while ( !importService.getState( importUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
Thread.sleep(100);
}
@@ -431,8 +451,8 @@ public class ImportCollectionIT {
}});
}});
- int maxRetries = 0;
- int retries = 100;
+ int maxRetries = 100;
+ int retries = 0;
while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
Thread.sleep(100);
}
@@ -443,50 +463,50 @@ public class ImportCollectionIT {
* Create test entities of a specified type.
* First two entities are connected.
*/
- private void createTestEntities( final EntityManager em,
- Map<UUID, Entity> thingsMap, List<Entity> things, final String type) throws Exception {
+ private void createTestEntities(final EntityManager em,
+ Map<UUID, Entity> thingsMap, List<Entity> things, final String type) throws Exception {
logger.debug("\n\nCreating new {} collection in application {}\n",
- type, em.getApplication().getName() );
+ type, em.getApplication().getName());
em.refreshIndex();
List<Entity> created = new ArrayList<>();
- for ( int i = 0; i < 10; i++ ) {
+ for (int i = 0; i < 10; i++) {
final int count = i;
- Entity e = em.create( type, new HashMap<String, Object>() {{
+ Entity e = em.create(type, new HashMap<String, Object>() {{
put("name", em.getApplication().getName() + "-" + type + "-" + count);
put("originalAppId", em.getApplication().getUuid());
put("originalAppName", em.getApplication().getName());
}});
thingsMap.put(e.getUuid(), e);
- things.add( e );
- created.add( e );
+ things.add(e);
+ created.add(e);
}
// first two things are related to each other
- em.createConnection( new SimpleEntityRef( type, created.get(0).getUuid()),
- "related", new SimpleEntityRef( type, created.get(1).getUuid()));
- em.createConnection( new SimpleEntityRef( type, created.get(1).getUuid()),
- "related", new SimpleEntityRef( type, created.get(0).getUuid()));
+ em.createConnection(new SimpleEntityRef(type, created.get(0).getUuid()),
+ "related", new SimpleEntityRef(type, created.get(1).getUuid()));
+ em.createConnection(new SimpleEntityRef(type, created.get(1).getUuid()),
+ "related", new SimpleEntityRef(type, created.get(0).getUuid()));
em.refreshIndex();
- }
+ }
/**
* Delete the configured s3 bucket.
*/
- public void deleteBucket() {
+ public void deleteBucket() {
logger.debug("\n\nDelete bucket\n");
- String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
- String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
+ String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
+ String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);
Properties overrides = new Properties();
- overrides.setProperty( "s3" + ".identity", accessId );
- overrides.setProperty( "s3" + ".credential", secretKey );
+ overrides.setProperty("s3" + ".identity", accessId);
+ overrides.setProperty("s3" + ".credential", secretKey);
final Iterable<? extends Module> MODULES = ImmutableSet
.of(new JavaUrlHttpCommandExecutorServiceModule(),
@@ -494,11 +514,49 @@ public class ImportCollectionIT {
new NettyPayloadModule());
BlobStoreContext context =
- ContextBuilder.newBuilder("s3").credentials( accessId, secretKey ).modules( MODULES )
- .overrides( overrides ).buildView( BlobStoreContext.class );
+ ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
+ .overrides(overrides).buildView(BlobStoreContext.class);
BlobStore blobStore = context.getBlobStore();
blobStore.deleteContainer( bucketName );
- }
+ }
+
+
+ private static void deleteBucketsWithPrefix() {
+
+ logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix );
+
+ String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
+ String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);
+
+ Properties overrides = new Properties();
+ overrides.setProperty("s3" + ".identity", accessId);
+ overrides.setProperty("s3" + ".credential", secretKey);
+
+ final Iterable<? extends Module> MODULES = ImmutableSet
+ .of(new JavaUrlHttpCommandExecutorServiceModule(),
+ new Log4JLoggingModule(),
+ new NettyPayloadModule());
+
+ BlobStoreContext context =
+ ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
+ .overrides(overrides).buildView(BlobStoreContext.class);
+
+ BlobStore blobStore = context.getBlobStore();
+ final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();
+
+ for ( Object o : blobStoreList.toArray() ) {
+ StorageMetadata s = (StorageMetadata)o;
+
+ if ( s.getName().startsWith( bucketPrefix )) {
+ try {
+ blobStore.deleteContainer(s.getName());
+ } catch ( ContainerNotFoundException cnfe ) {
+ logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe );
+ }
+ logger.debug("Deleted bucket {}", s.getName());
+ }
+ }
+ }
}
[3/4] incubator-usergrid git commit: Fixed to export tests.
Posted by sn...@apache.org.
Fixed to export tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/be1b9ff5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/be1b9ff5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/be1b9ff5
Branch: refs/heads/two-dot-o-import
Commit: be1b9ff57438fd5916c1f0ae9f5bcb6d570470b8
Parents: b3c3e94
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Feb 5 13:59:37 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Feb 5 13:59:37 2015 -0500
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 15 +-
.../management/export/ExportServiceIT.java | 302 +++++++++----------
.../management/export/MockS3ExportImpl.java | 4 +-
.../management/importer/ImportCollectionIT.java | 9 +-
4 files changed, 153 insertions(+), 177 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/be1b9ff5/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index bdb28d3..e27857f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -338,7 +338,7 @@ public class ExportServiceImpl implements ExportService {
continue;
}
- appFileName = prepareOutputFileName( "application", application.getValue(), null );
+ appFileName = prepareOutputFileName( application.getValue(), null );
File ephemeral = collectionExportAndQuery( application.getKey(), config, export, jobExecution );
@@ -371,9 +371,9 @@ public class ExportServiceImpl implements ExportService {
Export export = getExportEntity( jobExecution );
ApplicationInfo application = managementService.getApplicationInfo( applicationId );
- String appFileName = prepareOutputFileName( "application", application.getName(), null );
+ String appFileName = prepareOutputFileName( application.getName(), null );
- File ephemeral = collectionExportAndQuery( applicationId, config, export, jobExecution );
+ File ephemeral = collectionExportAndQuery(applicationId, config, export, jobExecution);
fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
@@ -390,8 +390,7 @@ public class ExportServiceImpl implements ExportService {
Export export = getExportEntity( jobExecution );
ApplicationInfo application = managementService.getApplicationInfo( applicationUUID );
- String appFileName = prepareOutputFileName( "application", application.getName(),
- ( String ) config.get( "collectionName" ) );
+ String appFileName = prepareOutputFileName( application.getName(), ( String ) config.get( "collectionName" ) );
File ephemeral = collectionExportAndQuery( applicationUUID, config, export, jobExecution );
@@ -536,13 +535,11 @@ public class ExportServiceImpl implements ExportService {
/**
- * @param type just a label such us: organization, application.
- *
* @return the file name concatenated with the type and the name of the collection
*/
- public String prepareOutputFileName( String type, String name, String CollectionName ) {
+ public String prepareOutputFileName( String applicationName, String CollectionName ) {
StringBuilder str = new StringBuilder();
- str.append( name );
+ str.append( applicationName );
str.append( "." );
if ( CollectionName != null ) {
str.append( CollectionName );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/be1b9ff5/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
index d23d358..7cb41e7 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
@@ -20,14 +20,12 @@ package org.apache.usergrid.management.export;
import java.io.File;
import java.io.FileReader;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
+import java.util.*;
+import com.google.common.util.concurrent.Service;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.usergrid.batch.service.JobSchedulerService;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
@@ -106,6 +104,11 @@ public class ExportServiceIT {
public void setup() throws Exception {
logger.info("in setup");
+ // start the scheduler after we're all set up
+ JobSchedulerService jobScheduler = cassandraResource.getBean( JobSchedulerService.class );
+ if ( jobScheduler.state() != Service.State.RUNNING ) {
+ jobScheduler.startAndWait();
+ }
adminUser = orgAppAdminRule.getAdminInfo();
organization = orgAppAdminRule.getOrganizationInfo();
@@ -151,7 +154,9 @@ public class ExportServiceIT {
S3Export s3Export = new MockS3ExportImpl("testFileConnections.json" );
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
@@ -180,14 +185,6 @@ public class ExportServiceIT {
UUID exportUUID = exportService.schedule( payload );
- //create and initialize jobData returned in JobExecution.
- JobData jobData = jobDataCreator( payload,exportUUID,s3Export );
-
- JobExecution jobExecution = mock( JobExecution.class );
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- exportService.doExport( jobExecution );
-
JSONParser parser = new JSONParser();
org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
@@ -224,7 +221,8 @@ public class ExportServiceIT {
f = new File( "testConnectionsOnApplicationEndpoint.json" );
}
catch ( Exception e ) {
- //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
+ // consumed because this checks to see if the file exists.
+ // If it doesn't then don't do anything and carry on.
}
String fileName = "testConnectionsOnApplicationEndpoint.json";
@@ -232,7 +230,9 @@ public class ExportServiceIT {
S3Export s3Export = new MockS3ExportImpl( "testConnectionsOnApplicationEndpoint.json" );
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
@@ -308,24 +308,30 @@ public class ExportServiceIT {
f = new File( "exportOneOrg.json" );
}
catch ( Exception e ) {
- //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
+ //consumed because this checks to see if the file exists.
+ // If it doesn't then don't do anything and carry on.
}
//create another org to ensure we don't export it
- orgAppAdminRule.createOwnerAndOrganization( "noExport"+newUUIDString(), "junkUserName"+newUUIDString(), "junkRealName"+newUUIDString(), newUUIDString()+"ugExport@usergrid.com",
- "123456789" );
+ orgAppAdminRule.createOwnerAndOrganization(
+ "noExport"+newUUIDString(),
+ "junkUserName"+newUUIDString(),
+ "junkRealName"+newUUIDString(),
+ newUUIDString()+"ugExport@usergrid.com",
+ "123456789" );
S3Export s3Export = new MockS3ExportImpl("exportOneOrg.json");
// s3Export.setFilename( "exportOneOrg.json" );
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
payload.put( "collectionName", "roles" );
UUID exportUUID = exportService.schedule( payload );
- //exportService.setS3Export( s3Export );
JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
@@ -391,7 +397,8 @@ public class ExportServiceIT {
S3Export s3Export = new MockS3ExportImpl("exportOneApp.json");
//s3Export.setFilename( "exportOneApp.json" );
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+
+ HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
@@ -449,7 +456,9 @@ public class ExportServiceIT {
S3Export s3Export = new MockS3ExportImpl("exportOneAppWQuery.json" );
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "query", "select * where username = 'junkRealName'" );
payload.put( "organizationId", organization.getUuid() );
@@ -511,7 +520,9 @@ public class ExportServiceIT {
S3Export s3Export = new MockS3ExportImpl("exportOneCollection.json" );
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
@@ -571,7 +582,9 @@ public class ExportServiceIT {
S3Export s3Export = new MockS3ExportImpl("exportOneCollectionWQuery.json");
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
payload.put( "query", "select * where username contains 'billybob0'" );
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
@@ -600,90 +613,78 @@ public class ExportServiceIT {
@Test
public void testExportOneOrganization() throws Exception {
- int entitiesToCreate = 20;
- File f = null;
+ // create a bunch of organizations, each with applications and collections of entities
+ int maxOrgs = 3;
+ int maxApps = 3;
+ int maxEntities = 20;
- try {
- f = new File( "exportOneOrganization.json" );
- }
- catch ( Exception e ) {
- // consumed because this checks to see if the file exists.
- // If it doesn't, don't do anything and carry on.
- }
+ List<ApplicationInfo> appsMade = new ArrayList<>();
+ List<OrganizationInfo> orgsMade = new ArrayList<>();
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
- em.createApplicationCollection( "newOrg" );
- //intialize user object to be posted
- Map<String, Object> userProperties = null;
- Entity[] entity;
- entity = new Entity[entitiesToCreate];
- //creates entities
- for ( int i = 0; i < entitiesToCreate; i++ ) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test" + i + "@anuff.com" );
- entity[i] = em.create( "newOrg", userProperties );
- }
+ for ( int orgIndex = 0; orgIndex < maxOrgs; orgIndex++ ) {
- S3Export s3Export = new MockS3ExportImpl("exportOneOrganization.json" );
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+ String orgName = "org_" + RandomStringUtils.randomAlphanumeric(10);
+ OrganizationInfo orgMade = setup.getMgmtSvc().createOrganization( orgName, adminUser, true );
+ orgsMade.add( orgMade );
+ logger.debug("Created org {}", orgName);
- // creates 100s of organizations with some entities
- // in each one to make sure we don't actually apply it
- final String uniqueOrg = uniqueOrg();
- final String uniqueApp = uniqueApp();
+ for ( int appIndex = 0; appIndex < maxApps; appIndex++ ) {
- OrganizationInfo orgMade = null;
- ApplicationInfo appMade = null;
- for ( int i = 0; i < 10; i++ ) {
- orgMade = setup.getMgmtSvc().createOrganization(uniqueOrg + i, adminUser, true );
- appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), uniqueApp + i );
+ String appName = "app_" + RandomStringUtils.randomAlphanumeric(10);
+ ApplicationInfo appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), appName );
+ appsMade.add( appMade );
+ logger.debug("Created app {}", appName);
- EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
- customMaker.createApplicationCollection( "superappCol" + i );
- //intialize user object to be posted
- Map<String, Object> entityLevelProperties = null;
- Entity[] entNotCopied;
- entNotCopied = new Entity[entitiesToCreate];
- //creates entities
- for ( int index = 0; index < 20; index++ ) {
- entityLevelProperties = new LinkedHashMap<String, Object>();
- entityLevelProperties.put( "username", "bobso" + index );
- entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
- entNotCopied[index] = customMaker.create( "superappCol", entityLevelProperties );
+ for (int entityIndex = 0; entityIndex < maxEntities; entityIndex++) {
+
+ EntityManager appEm = setup.getEmf().getEntityManager( appMade.getId() );
+ appEm.create( appName + "_type", new HashMap<String, Object>() {{
+ put("property1", "value1");
+ put("property2", "value2");
+ }});
+ }
}
}
- payload.put( "organizationId", orgMade.getUuid() );
- payload.put( "applicationId", appMade.getId() );
- UUID exportUUID = exportService.schedule( payload );
+ // export one of the organizations only, using mock S3 export that writes to local disk
- JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
+ String exportFileName = "exportOneOrganization.json";
+ S3Export s3Export = new MockS3ExportImpl( exportFileName );
+
+ HashMap<String, Object> payload = payloadBuilder(appsMade.get(0).getName());
+ payload.put("organizationId", orgsMade.get(0).getUuid() );
+ payload.put( "applicationId", appsMade.get(0).getId() );
+
+ ExportService exportService = setup.getExportService();
+ UUID exportUUID = exportService.schedule( payload );
+ JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
- Thread.sleep(1000);
-
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ // finally, we check that file was created and contains the right number of entities in the array
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ File exportedFile = new File( exportFileName );
+ exportedFile.deleteOnExit();
+
+ JSONParser parser = new JSONParser();
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray )
+ parser.parse( new FileReader( exportedFile ) );
- /*plus 3 for the default roles*/
assertEquals( 23, a.size() );
- f.deleteOnExit();
}
@Test
public void testExportDoJob() throws Exception {
- HashMap<String, Object> payload = payloadBuilder();
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
@@ -697,6 +698,7 @@ public class ExportServiceIT {
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
+ when( jobExecution.getJobId() ).thenReturn( UUID.randomUUID() );
ExportJob job = new ExportJob();
ExportService eS = mock( ExportService.class );
@@ -705,6 +707,7 @@ public class ExportServiceIT {
job.doJob( jobExecution );
}
catch ( Exception e ) {
+ logger.error("Error doing job", e);
assert ( false );
}
assert ( true );
@@ -719,6 +722,7 @@ public class ExportServiceIT {
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
+ when( jobExecution.getJobId() ).thenReturn( UUID.randomUUID() );
ExportJob job = new ExportJob();
S3Export s3Export = mock( S3Export.class );
@@ -742,6 +746,7 @@ public class ExportServiceIT {
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
+ when( jobExecution.getJobId() ).thenReturn( UUID.randomUUID() );
ExportJob job = new ExportJob();
S3Export s3Export = mock( S3Export.class );
@@ -760,97 +765,89 @@ public class ExportServiceIT {
@Test
public void testIntegration100EntitiesOn() throws Exception {
- S3Export s3Export = new S3ExportImpl();
+ logger.debug("testIntegration100EntitiesOn(): starting...");
+
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
-
- //intialize user object to be posted
- ApplicationInfo appMade = null;
+ // create five applications each with collection of five entities
for ( int i = 0; i < 5; i++ ) {
- appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp" + i );
- EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
+ ApplicationInfo appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp" + i );
+ EntityManager appEm = setup.getEmf().getEntityManager( appMade.getId() );
String collName = "superappCol" + i;
- customMaker.createApplicationCollection( collName );
+ appEm.createApplicationCollection(collName);
- //intialize user object to be posted
Map<String, Object> entityLevelProperties = null;
Entity[] entNotCopied;
entNotCopied = new Entity[5];
- //creates entities
for ( int index = 0; index < 5; index++ ) {
entityLevelProperties = new LinkedHashMap<String, Object>();
entityLevelProperties.put( "username", "bobso" + index );
entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
- entNotCopied[index] = customMaker.create( collName, entityLevelProperties );
+ entNotCopied[index] = appEm.create( collName, entityLevelProperties );
}
}
- UUID exportUUID = exportService.schedule( payload );
-
- //create and initialize jobData returned in JobExecution.
- JobData jobData = jobDataCreator( payload,exportUUID, s3Export );
+ // export the organization containing those apps and collections
- JobExecution jobExecution = mock( JobExecution.class );
- when( jobExecution.getJobData() ).thenReturn( jobData );
+ UUID exportUUID = exportService.schedule( payload );
- exportService.doExport( jobExecution );
- while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {
- ;
+ int maxRetries = 0;
+ int retries = 100;
+ while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
+ Thread.sleep(100);
}
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
-
Properties overrides = new Properties();
overrides.setProperty( "s3" + ".identity", accessId );
overrides.setProperty( "s3" + ".credential", secretKey );
- Blob bo = null;
- BlobStore blobStore = null;
+ // test that we can find the file that were exported to S3
+ BlobStore blobStore = null;
try {
+
final Iterable<? extends Module> MODULES = ImmutableSet.of(
new JavaUrlHttpCommandExecutorServiceModule(),
new Log4JLoggingModule(),
- new NettyPayloadModule() );
+ new NettyPayloadModule());
- BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
- .credentials(accessId, secretKey )
- .modules(MODULES )
- .overrides(overrides )
- .buildView(BlobStoreContext.class );
+ BlobStoreContext context = ContextBuilder.newBuilder("s3")
+ .credentials(accessId, secretKey)
+ .modules(MODULES)
+ .overrides(overrides)
+ .buildView(BlobStoreContext.class);
- String expectedFileName = ((ExportServiceImpl)exportService)
- .prepareOutputFileName("organization", organization.getName(), "applications");
+ String expectedFileName = ((ExportServiceImpl) exportService)
+ .prepareOutputFileName(organization.getName(), "applications");
blobStore = context.getBlobStore();
- if ( !blobStore.blobExists( bucketName, expectedFileName ) ) {
- blobStore.deleteContainer( bucketName );
- assert ( false );
+ if (!blobStore.blobExists(bucketName, expectedFileName)) {
+ blobStore.deleteContainer(bucketName);
+ Assert.fail("Blob does not exist: " + expectedFileName);
}
- bo = blobStore.getBlob( bucketName, expectedFileName );
+ Blob bo = blobStore.getBlob(bucketName, expectedFileName);
- Long numOfFiles = blobStore.countBlobs( bucketName );
+ Long numOfFiles = blobStore.countBlobs(bucketName);
Long numWeWant = 1L;
- blobStore.deleteContainer( bucketName );
- assertEquals( numOfFiles, numWeWant );
+ blobStore.deleteContainer(bucketName);
+ assertEquals(numOfFiles, numWeWant);
+ assertNotNull(bo);
+ } finally {
+ blobStore.deleteContainer(bucketName);
}
- catch ( Exception e ) {
- assert ( false );
- }
-
- assertNotNull( bo );
- blobStore.deleteContainer( bucketName );
}
@Ignore("Why is this ignored?")
@@ -859,7 +856,9 @@ public class ExportServiceIT {
S3Export s3Export = new S3ExportImpl();
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
OrganizationInfo orgMade = null;
ApplicationInfo appMade = null;
@@ -888,14 +887,6 @@ public class ExportServiceIT {
UUID exportUUID = exportService.schedule( payload );
assertNotNull( exportUUID );
- //create and initialize jobData returned in JobExecution.
- JobData jobData = jobDataCreator( payload,exportUUID,s3Export );
-
- JobExecution jobExecution = mock( JobExecution.class );
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- exportService.doExport( jobExecution );
-
Thread.sleep( 3000 );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
@@ -905,7 +896,6 @@ public class ExportServiceIT {
overrides.setProperty( "s3" + ".identity", accessId );
overrides.setProperty( "s3" + ".credential", secretKey );
- Blob bo = null;
BlobStore blobStore = null;
try {
@@ -926,23 +916,20 @@ public class ExportServiceIT {
Long numOfFiles = blobStore.countBlobs( bucketName );
String expectedFileName = ((ExportServiceImpl)exportService)
- .prepareOutputFileName("organization", organization.getName(), "applications");
+ .prepareOutputFileName(organization.getName(), "applications");
//delete container containing said files
- bo = blobStore.getBlob(bucketName, expectedFileName);
+ Blob bo = blobStore.getBlob(bucketName, expectedFileName);
Long numWeWant = 5L;
blobStore.deleteContainer( bucketName );
//asserts that the correct number of files was transferred over
assertEquals( numWeWant, numOfFiles );
+
}
- catch ( Exception e ) {
+ finally {
blobStore.deleteContainer( bucketName );
- e.printStackTrace();
- assert ( false );
}
-
- assertNotNull( bo );
}
@@ -952,7 +939,9 @@ public class ExportServiceIT {
S3Export s3Export = new S3ExportImpl();
ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
+
+ String appName = orgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
payload.put( "organizationId", organization.getUuid() );
payload.put( "applicationId", applicationId );
@@ -979,10 +968,12 @@ public class ExportServiceIT {
}
EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
//intialize user object to be posted
Map<String, Object> userProperties = null;
Entity[] entity;
entity = new Entity[100];
+
//creates entities
for ( int i = 0; i < 100; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
@@ -993,19 +984,8 @@ public class ExportServiceIT {
}
UUID exportUUID = exportService.schedule( payload );
- // exportService.setS3Export( s3Export );
- //create and initialize jobData returned in JobExecution.
- JobData jobData = jobDataCreator( payload,exportUUID,s3Export );
-
-
- JobExecution jobExecution = mock( JobExecution.class );
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- exportService.doExport( jobExecution );
- while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {
- ;
- }
+ while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {}
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
@@ -1030,7 +1010,7 @@ public class ExportServiceIT {
.buildView(BlobStoreContext.class );
String expectedFileName = ((ExportServiceImpl)exportService)
- .prepareOutputFileName("organization", organization.getName(), "applications");
+ .prepareOutputFileName(organization.getName(), "applications");
blobStore = context.getBlobStore();
if ( !blobStore.blobExists( bucketName, expectedFileName ) ) {
@@ -1062,7 +1042,7 @@ public class ExportServiceIT {
}
/*Creates fake payload for testing purposes.*/
- public HashMap<String, Object> payloadBuilder() {
+ public HashMap<String, Object> payloadBuilder( String orgOrAppName ) {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> properties = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
@@ -1070,12 +1050,12 @@ public class ExportServiceIT {
System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
storage_info.put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ) );
- storage_info.put( "bucket_location", System.getProperty( "bucketName" ) );
+ storage_info.put( "bucket_location", bucketName );
properties.put( "storage_provider", "s3" );
properties.put( "storage_info", storage_info );
- payload.put( "path", "test-organization/test-app" );
+ payload.put( "path", orgOrAppName );
payload.put( "properties", properties );
return payload;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/be1b9ff5/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
index 8fc4b58..eeab57c 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
@@ -40,14 +40,14 @@ public class MockS3ExportImpl implements S3Export {
this.filename = filename;
}
-
+
@Override
public void copyToS3( File ephemeral, final Map<String,Object> exportInfo, String ignoredFileName ) {
File verifiedData = new File( filename );
try {
FileUtils.copyFile(ephemeral, verifiedData);
- logger.info("Copied file {} to {}", filename, ephemeral.getAbsolutePath());
+ logger.info( "Copied file {} to {}", ephemeral.getAbsolutePath(), verifiedData );
}
catch ( IOException e ) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/be1b9ff5/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index 4297443..89101e5 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -443,7 +443,7 @@ public class ImportCollectionIT {
* Create test entities of a specified type.
* First two entities are connected.
*/
- private void createTestEntities( final EntityManager em,
+ private void createTestEntities( final EntityManager em,
Map<UUID, Entity> thingsMap, List<Entity> things, final String type) throws Exception {
logger.debug("\n\nCreating new {} collection in application {}\n",
@@ -471,13 +471,13 @@ public class ImportCollectionIT {
"related", new SimpleEntityRef( type, created.get(0).getUuid()));
em.refreshIndex();
- }
+ }
/**
* Delete the configured s3 bucket.
*/
- public void deleteBucket() {
+ public void deleteBucket() {
logger.debug("\n\nDelete bucket\n");
@@ -499,7 +499,6 @@ public class ImportCollectionIT {
BlobStore blobStore = context.getBlobStore();
blobStore.deleteContainer( bucketName );
- }
-
+ }
}