You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/10 21:26:19 UTC
[26/50] [abbrv] git commit: Updated endpoint such that the user must
now specify ( through the endpoint ) what collection they want to backup.
Updated tests to reflect the collection lookup Added collection name to
export info.
Updated endpoint such that the user must now specify ( through the endpoint ) what collection they want to backup.
Updated tests to reflect the collection lookup
Added collection name to export info.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/88a2a485
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/88a2a485
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/88a2a485
Branch: refs/pull/70/merge
Commit: 88a2a485037f2c324c975fbcfb6cca5d5c30ae98
Parents: 1a2028d
Author: grey <gr...@apigee.com>
Authored: Thu Feb 27 15:41:06 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Thu Feb 27 15:41:06 2014 -0800
----------------------------------------------------------------------
.../applications/ApplicationResource.java | 9 +-
.../rest/management/ManagementResourceIT.java | 8 +-
.../apache/usergrid/management/ExportInfo.java | 6 +-
.../management/export/ExportServiceImpl.java | 123 ++++++++++++-------
.../cassandra/ManagementServiceIT.java | 77 +++++++++++-
5 files changed, 172 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index e94f4df..0e9d8ed 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -212,22 +212,24 @@ public class ApplicationResource extends AbstractContextResource {
@POST
- @Path("export")
+ @Path("collection/{collection_name}/export")
@Consumes(APPLICATION_JSON)
@RequireOrganizationAccess
- public Response exportPostJson( @Context UriInfo ui, Map<String, Object> json,
+ public Response exportPostJson( @Context UriInfo ui,@PathParam( "collection_name" ) String collection_name ,Map<String, Object> json,
@QueryParam("callback") @DefaultValue("") String callback )
throws OAuthSystemException {
OAuthResponse response = null;
UUID jobUUID = null;
+ String colExport = collection_name;
Map<String, String> uuidRet = new HashMap<String, String>();
try {
//parse the json into some useful object (the config params)
ExportInfo objEx = new ExportInfo( json );
objEx.setApplicationId( applicationId );
+ objEx.setCollection( colExport );
jobUUID = exportService.schedule( objEx );
uuidRet.put( "jobUUID", jobUUID.toString() );
}
@@ -250,6 +252,7 @@ public class ApplicationResource extends AbstractContextResource {
@GET
+ @RequireOrganizationAccess
@Path("export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}")
public Response exportGetJson( @Context UriInfo ui, @PathParam("jobUUID") UUID jobUUIDStr,
@QueryParam("callback") @DefaultValue("") String callback ) throws Exception {
@@ -269,4 +272,6 @@ public class ApplicationResource extends AbstractContextResource {
return Response.status( SC_OK ).entity( entity.getState() ).build();
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index a36dfd3..a81d9fb 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -541,7 +541,7 @@ public class ManagementResourceIT extends AbstractRestIT {
HashMap<String, Object> payload = payloadBuilder();
try {
- node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/collection/users/export" )
.queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
}
@@ -606,7 +606,7 @@ public class ManagementResourceIT extends AbstractRestIT {
HashMap<String, Object> payload = payloadBuilder();
try {
- node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/collection/users/export" )
.queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
}
@@ -628,7 +628,7 @@ public class ManagementResourceIT extends AbstractRestIT {
HashMap<String, Object> payload = payloadBuilder();
- node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/collection/users/export" )
.queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
String uuid = String.valueOf( node.get( "jobUUID" ) );
@@ -673,7 +673,7 @@ public class ManagementResourceIT extends AbstractRestIT {
Status responseStatus = Status.OK;
UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
try {
- node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + fake )
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/collection/users/export/" + fake )
.accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
.get( JsonNode.class );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
index 4436f0b..b1be504 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
@@ -20,7 +20,8 @@ public class ExportInfo extends TypedEntity {
private String s3_accessId;
private String s3_key;
private String bucket_location;
- private UUID applicationId;
+ private UUID applicationId; //TODO: include application when initializing export info
+ private String collection; //TODO: include collection when initializing export info
public ExportInfo( Map<String, Object> exportData ) {
@@ -33,6 +34,7 @@ public class ExportInfo extends TypedEntity {
bucket_location = ( String ) storage_info.get( "bucket_location" );
}
+ public String getCollection() { return collection; }
public UUID getApplicationId() {
return applicationId;
@@ -70,4 +72,6 @@ public class ExportInfo extends TypedEntity {
public void setApplicationId( UUID appId ) { applicationId = appId;}
+
+ public void setCollection ( String colName) { collection = colName; }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 37646a5..b716293 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -75,7 +75,7 @@ public class ExportServiceImpl implements ExportService {
@Override
public UUID schedule( final ExportInfo config ) throws Exception {
- if(config == null){
+ if ( config == null ) {
logger.error( "export information cannot be null" );
return null;
}
@@ -84,18 +84,22 @@ public class ExportServiceImpl implements ExportService {
Export export = new Export();
- //write to the em
+ //update state
export = em.create( export );
export.setState( Export.State.CREATED );
em.update( export );
+ //set data to be transferred to exportInfo
JobData jobData = new JobData();
jobData.setProperty( "exportInfo", config );
jobData.setProperty( EXPORT_ID, export.getUuid() );
long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
+ //schedule job
sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
+
+ //update state
export.setState( Export.State.SCHEDULED );
em.update( export );
@@ -104,14 +108,17 @@ public class ExportServiceImpl implements ExportService {
/**
- * Query Entity Manager for specific Export Entity within application
- *
+ * Query Entity Manager for the string state of the Export Entity.
+ * This corresponds to the GET /export
* @return String
*/
@Override
public String getState( final UUID appId, final UUID uuid ) throws Exception {
+ //get application entity manager
EntityManager rootEm = emf.getEntityManager( appId );
+
+ //retrieve the export entity.
Export export = rootEm.get( uuid, Export.class );
if ( export == null ) {
@@ -124,19 +131,26 @@ public class ExportServiceImpl implements ExportService {
@Override
public void doExport( final ExportInfo config, final JobExecution jobExecution ) throws Exception {
+ //get the entity manager for the application, and the entity that this Export corresponds to.
UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
EntityManager em = emf.getEntityManager( config.getApplicationId() );
Export export = em.get( exportId, Export.class );
+ //update the entity state to show that the job has officially started.
export.setState( Export.State.STARTED );
em.update( export );
+ //retrieves all the organizations. Loops through them and backs up the appplications for each one
Map<UUID, String> organizationGet = getOrgs( config );
for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
try {
+ //exports all the applications for a single organization
exportApplicationsForOrg( organization, config, jobExecution );
- }catch(Exception e){
+ }
+ catch ( Exception e ) {
+ //if for any reason the backing up fails, then update the entity with a failed state.
export.setState( Export.State.FAILED );
+ em.update( export );
return;
}
}
@@ -145,6 +159,12 @@ public class ExportServiceImpl implements ExportService {
}
+ /**
+ * Loops through all the organizations and returns a Map with the corresponding information
+ * @param exportInfo
+ * @return Map<UUID, String>
+ * @throws Exception
+ */
private Map<UUID, String> getOrgs( ExportInfo exportInfo ) throws Exception {
// Loop through the organizations
UUID orgId = null;
@@ -204,13 +224,24 @@ public class ExportServiceImpl implements ExportService {
//write test checking to see what happens if the input stream is closed or wrong.
//TODO: make multipart streaming functional
//currently only stores the collection in memory then flushes it.
+
+
+ /**
+ * Exports all applications for the given organization.
+ * @param organization
+ * @param config
+ * @param jobExecution
+ * @throws Exception
+ */
private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config,
final JobExecution jobExecution ) throws Exception {
+ //retrieves export entity
UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
EntityManager exportManager = emf.getEntityManager( config.getApplicationId() );
Export export = exportManager.get( exportId, Export.class );
+ //sets up a output stream for s3 backup.
ByteArrayOutputStream baos = new ByteArrayOutputStream();
logger.info( "" + organization );
@@ -236,26 +267,25 @@ public class ExportServiceImpl implements ExportService {
// through the entities in the application (former namespace).
//could support queries, just need to implement that in the rest endpoint.
for ( String collectionName : metadata.keySet() ) {
- if ( collectionName.equals( "exports" ) ) {
- continue;
- }
-
- Query query = new Query();
- query.setLimit( MAX_ENTITY_FETCH );
- query.setResultsLevel( Results.Level.ALL_PROPERTIES );
- Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
-
- PagingResultsIterator itr = new PagingResultsIterator( entities );
-
- for ( Object e : itr ) {
- starting_time = checkTimeDelta( starting_time, jobExecution );
- Entity entity = ( Entity ) e;
-
- jg.writeStartObject();
- jg.writeFieldName( "Metadata" );
- jg.writeObject( entity );
- saveCollectionMembers( jg, em, application.getValue(), entity );
- jg.writeEndObject();
+ //if the collection you are looping through doesn't match the name of the one you want. Don't export it.
+ if ( collectionName.equals( config.getCollection() ) ) {
+ //Query entity manager for the entities in a collection
+ Query query = new Query();
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+ Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
+
+ //pages through the query and backs up all results.
+ PagingResultsIterator itr = new PagingResultsIterator( entities );
+ for ( Object e : itr ) {
+ starting_time = checkTimeDelta( starting_time, jobExecution );
+ Entity entity = ( Entity ) e;
+ jg.writeStartObject();
+ jg.writeFieldName( "Metadata" );
+ jg.writeObject(entity );
+ saveCollectionMembers( jg, em, config.getCollection(), entity );
+ jg.writeEndObject();
+ }
}
}
@@ -265,10 +295,12 @@ public class ExportServiceImpl implements ExportService {
baos.flush();
baos.close();
+ //sets up the Inputstream for copying the method to s3.
InputStream is = new ByteArrayInputStream( baos.toByteArray() );
try {
s3Export.copyToS3( is, config, appFileName );
- }catch(Exception e){
+ }
+ catch ( Exception e ) {
export.setState( Export.State.FAILED );
return;
}
@@ -276,6 +308,12 @@ public class ExportServiceImpl implements ExportService {
}
+ /**
+ * Regulates how long to wait until the next heartbeat.
+ * @param startingTime
+ * @param jobExecution
+ * @return
+ */
public long checkTimeDelta( long startingTime, final JobExecution jobExecution ) {
long cur_time = System.currentTimeMillis();
@@ -292,38 +330,40 @@ public class ExportServiceImpl implements ExportService {
* Serialize and save the collection members of this <code>entity</code>
*
* @param em Entity Manager
- * @param application Application name
+ * @param collection Collection Name
* @param entity entity
*/
- private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String application, Entity entity )
+ private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String collection, Entity entity )
throws Exception {
Set<String> collections = em.getCollections( entity );
- // Only create entry for Entities that have collections
+ // If your application doesn't have any e
if ( ( collections == null ) || collections.isEmpty() ) {
return;
}
for ( String collectionName : collections ) {
- jg.writeFieldName( collectionName );
- jg.writeStartArray();
+ if ( collectionName.equals( collection ) ) {
+ jg.writeFieldName( collectionName );
+ jg.writeStartArray();
- //is 100000 an arbitary number?
- Results collectionMembers =
- em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false );
+ //is 100000 an arbitary number?
+ Results collectionMembers =
+ em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false );
- List<UUID> entityIds = collectionMembers.getIds();
+ List<UUID> entityIds = collectionMembers.getIds();
- if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
- for ( UUID childEntityUUID : entityIds ) {
- jg.writeObject( childEntityUUID.toString() );
+ if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
+ for ( UUID childEntityUUID : entityIds ) {
+ jg.writeObject( childEntityUUID.toString() );
+ }
}
- }
- // End collection array.
- jg.writeEndArray();
+ // End collection array.
+ jg.writeEndArray();
+ }
}
// Write connections
@@ -331,7 +371,6 @@ public class ExportServiceImpl implements ExportService {
// Write dictionaries
saveDictionaries( entity, em, jg );
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/88a2a485/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index 0b1f7be..b4b82e0 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -766,6 +766,7 @@ public class ManagementServiceIT {
ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
+ exportInfo.setCollection( "users" );
EntityManager em = setup.getEmf().getEntityManager( applicationId );
//intialize user object to be posted
@@ -835,6 +836,7 @@ public class ManagementServiceIT {
ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
+ exportInfo.setCollection( "users" );
UUID exportUUID = exportService.schedule( exportInfo );
exportService.setS3Export( s3Export );
@@ -884,6 +886,7 @@ public class ManagementServiceIT {
ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
+ exportInfo.setCollection( "roles" );
UUID exportUUID = exportService.schedule( exportInfo );
exportService.setS3Export( s3Export );
@@ -902,7 +905,7 @@ public class ManagementServiceIT {
org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- //assertEquals( 3 , a.size() );
+ assertEquals( 3 , a.size() );
for ( int i = 0; i < a.size(); i++ ) {
org.json.simple.JSONObject entity = ( org.json.simple.JSONObject ) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
@@ -951,6 +954,7 @@ public class ManagementServiceIT {
ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
+ exportInfo.setCollection( "roles" );
UUID exportUUID = exportService.schedule( exportInfo );
exportService.setS3Export( s3Export );
@@ -969,7 +973,76 @@ public class ManagementServiceIT {
org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- //assertEquals( 3 , a.size() );
+ assertEquals( 3 , a.size() );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
+ String entityName = ( String ) entityData.get( "name" );
+ assertFalse( "junkRealName".equals( entityName ) );
+ //assertNotEquals( "NotEquals","junkRealName",entityName );
+ }
+ f.delete();
+ }
+
+ @Test
+ public void testFileExportOneCollection() throws Exception {
+
+ File f = null;
+// String orgName = "ed-organization";
+// String appName = "ed-app";
+
+ try {
+ f = new File( "test.json" );
+ f.delete();
+ }
+ catch ( Exception e ) {
+ //consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
+ }
+
+ //UUID appId = setup.getEmf().createApplication( orgName, appName );
+
+
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId);
+ em.createApplicationCollection( "baconators" );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[10];
+ //creates entities
+ for ( int i = 0; i < 10; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
+ entity[i] = em.create( "baconators", userProperties );
+ }
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo( payload );
+ exportInfo.setApplicationId( applicationId );
+ exportInfo.setCollection( "baconators" );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+
+ assertEquals( 10 , a.size() );
for ( int i = 0; i < a.size(); i++ ) {
org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );