You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/04/08 22:21:29 UTC
[02/40] git commit: Added organization level exports with a different
applications file for each application.
Added organization level exports with a different applications file for each application.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/844fd5d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/844fd5d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/844fd5d6
Branch: refs/heads/master
Commit: 844fd5d6eb1fc8416f4836d2abca30abc9d9394a
Parents: 2cd1d5d
Author: grey <gr...@apigee.com>
Authored: Mon Mar 31 00:32:57 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Mon Mar 31 00:32:57 2014 -0700
----------------------------------------------------------------------
.../organizations/OrganizationResource.java | 81 ++++++++++++
.../rest/management/ManagementResourceIT.java | 2 +-
.../management/export/ExportServiceImpl.java | 129 +++++++++++++++++--
.../cassandra/ManagementServiceIT.java | 89 +++++++------
4 files changed, 253 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/844fd5d6/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index b57d02a..05312ab 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -17,7 +17,9 @@
package org.apache.usergrid.rest.management.organizations;
+import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
import javax.ws.rs.Consumes;
import javax.ws.rs.DefaultValue;
@@ -29,20 +31,29 @@ import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.exception.OAuthSystemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+
import org.apache.usergrid.management.ActivationState;
import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.export.ExportService;
import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.applications.ServiceResource;
import org.apache.usergrid.rest.exceptions.RedirectionException;
import org.apache.usergrid.rest.management.organizations.applications.ApplicationsResource;
import org.apache.usergrid.rest.management.organizations.users.UsersResource;
import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+import org.apache.usergrid.rest.utils.JSONPUtils;
import org.apache.usergrid.security.oauth.ClientCredentialsInfo;
import org.apache.usergrid.security.tokens.exceptions.TokenException;
import org.apache.usergrid.services.ServiceResults;
@@ -50,6 +61,11 @@ import org.apache.usergrid.services.ServiceResults;
import com.sun.jersey.api.json.JSONWithPadding;
import com.sun.jersey.api.view.Viewable;
+import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
+import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
+
@Component("org.apache.usergrid.rest.management.organizations.OrganizationResource")
@Scope("prototype")
@@ -61,6 +77,9 @@ public class OrganizationResource extends AbstractContextResource {
private static final Logger logger = LoggerFactory.getLogger( OrganizationsResource.class );
+ @Autowired
+ protected ExportService exportService;
+
OrganizationInfo organization;
@@ -252,4 +271,66 @@ public class OrganizationResource extends AbstractContextResource {
return new JSONWithPadding( response, callback );
}
+
+ @POST
+ @Path("export")
+ @Consumes(APPLICATION_JSON)
+ @RequireOrganizationAccess
+ public Response exportPostJson( @Context UriInfo ui,Map<String, Object> json,
+ @QueryParam("callback") @DefaultValue("") String callback )
+ throws OAuthSystemException {
+
+
+ OAuthResponse response = null;
+ UUID jobUUID = null;
+ Map<String, String> uuidRet = new HashMap<String, String>();
+
+ Map<String,Object> properties;
+ Map<String, Object> storage_info;
+
+ try {
+ if((properties = ( Map<String, Object> ) json.get( "properties" )) == null){
+ throw new NullPointerException("Could not find 'properties'");
+ }
+ storage_info = ( Map<String, Object> ) properties.get( "storage_info" );
+ String storage_provider = ( String ) properties.get( "storage_provider" );
+ if(storage_provider == null) {
+ throw new NullPointerException( "Could not find field 'storage_provider'" );
+ }
+ if(storage_info == null) {
+ throw new NullPointerException( "Could not find field 'storage_info'" );
+ }
+
+
+ String bucketName = ( String ) storage_info.get( "bucket_location" );
+ String accessId = ( String ) storage_info.get( "s3_access_id" );
+ String secretKey = ( String ) storage_info.get( "s3_key" );
+
+ if(bucketName == null) {
+ throw new NullPointerException( "Could not find field 'bucketName'" );
+ }
+ if(accessId == null) {
+ throw new NullPointerException( "Could not find field 's3_access_id'" );
+ }
+ if(secretKey == null) {
+ throw new NullPointerException( "Could not find field 's3_key'" );
+ }
+
+ json.put( "organizationId",organization.getUuid());
+
+ jobUUID = exportService.schedule( json );
+ uuidRet.put( "Export Entity", jobUUID.toString() );
+ }
+ catch ( NullPointerException e ) {
+ return Response.status( SC_BAD_REQUEST ).type( JSONPUtils.jsonMediaType( callback ) )
+ .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build();
+ }
+ catch ( Exception e ) {
+ //TODO:throw descriptive error message and or include on in the response
+ //TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
+ return Response.status( SC_INTERNAL_SERVER_ERROR ).type( JSONPUtils.jsonMediaType( callback ) )
+ .entity( ServiceResource.wrapWithCallback( e.getMessage(), callback ) ).build();
+ }
+ return Response.status( SC_ACCEPTED ).entity( uuidRet ).build();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/844fd5d6/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index 9874bd4..d117e57 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -771,7 +771,7 @@ public class ManagementResourceIT extends AbstractRestIT {
uuid = uuid.replaceAll( "\"", "" );
try {
- node = resource().path( "/management/orgs/test-organization/export/" + uuid )
+ node = resource().path( "/management/orgs/test-organization/apps/exporters/export/" + uuid )
.queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/844fd5d6/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index b599854..053a4e3 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -48,6 +48,8 @@ import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.persistence.entities.JobData;
+import com.google.common.collect.BiMap;
+
/**
* Need to refactor out the mutliple orgs being take , need to factor out the multiple apps it will just be the one app
@@ -78,9 +80,12 @@ public class ExportServiceImpl implements ExportService {
private S3Export s3Export;
+ private String defaultAppExportname = "exporters";
+
@Override
public UUID schedule( final Map<String, Object> config ) throws Exception {
+ ApplicationInfo defaultExportApp = null;
if ( config == null ) {
logger.error( "export information cannot be null" );
@@ -88,8 +93,11 @@ public class ExportServiceImpl implements ExportService {
}
if ( config.get( "applicationId" ) == null ) {
- logger.error( "application information from export info could not be found" );
- return null;
+ defaultExportApp =
+ managementService.createApplication( ( UUID ) config.get( "organizationId" ), defaultAppExportname );
+ config.put( "applicationId", defaultExportApp.getId() );
+ //logger.error( "application information from export info could not be found" );
+ //return null;
}
EntityManager em = null;
@@ -169,24 +177,29 @@ public class ExportServiceImpl implements ExportService {
public void doExport( final JobExecution jobExecution ) throws Exception {
Map<String, Object> config = ( Map<String, Object> ) jobExecution.getJobData().getProperty( "exportInfo" );
+ UUID scopedAppId = ( UUID ) config.get( "applicationId" );
+
if ( config == null ) {
logger.error( "Export Information passed through is null" );
return;
}
//get the entity manager for the application, and the entity that this Export corresponds to.
UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
- if ( config.get( "applicationId" ) == null ) {
+ if ( scopedAppId == null ) {
logger.error( "Export Information application uuid is null" );
return;
}
- EntityManager em = emf.getEntityManager( ( UUID ) config.get( "applicationId" ) );
+ EntityManager em = emf.getEntityManager( scopedAppId );
Export export = em.get( exportId, Export.class );
//update the entity state to show that the job has officially started.
export.setState( Export.State.STARTED );
em.update( export );
- if ( config.get( "collectionName" ) == null ) {
+ if ( em.getApplication().getApplicationName().equals( "exporters" ) ) {
+ exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution );
+ }
+ else if ( config.get( "collectionName" ) == null ) {
//exports all the applications for a given organization.
if ( config.get( "organizationId" ) == null ) {
@@ -195,7 +208,7 @@ public class ExportServiceImpl implements ExportService {
em.update( export );
return;
}
- exportApplicationsForOrg( ( UUID ) config.get( "organizationId" ), ( UUID ) config.get( "applicationId" ),
+ exportApplicationFromOrg( ( UUID ) config.get( "organizationId" ), ( UUID ) config.get( "applicationId" ),
config, jobExecution );
}
else {
@@ -207,7 +220,7 @@ public class ExportServiceImpl implements ExportService {
em.update( export );
return;
}
- exportApplicationForOrg( ( UUID ) config.get( "organizationId" ),
+ exportCollectionFromOrgApp( ( UUID ) config.get( "organizationId" ),
( UUID ) config.get( "applicationId" ), config, jobExecution );
}
catch ( Exception e ) {
@@ -254,9 +267,99 @@ public class ExportServiceImpl implements ExportService {
/**
- * Exports all applications for the given organization.
+ * Exports All Applications from an Organization
*/
- private void exportApplicationsForOrg( UUID organizationUUID, UUID applicationId, final Map<String, Object> config,
+ private void exportApplicationsFromOrg( UUID organizationUUID, final Map<String, Object> config,
+ final JobExecution jobExecution ) throws Exception {
+
+ //retrieves export entity
+ UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
+ EntityManager exportManager = emf.getEntityManager( ( UUID ) config.get( "applicationId" ) );
+ Export export = exportManager.get( exportId, Export.class );
+ String appFileName = null;
+
+ //sets up a output stream for s3 backup.
+// ByteArrayOutputStream baos = new ByteArrayOutputStream();
+// JsonGenerator jg = getJsonGenerator( baos );
+
+ BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organizationUUID );
+
+ for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ JsonGenerator jg = getJsonGenerator( baos );
+ if(application.getValue().equals(
+ managementService.getOrganizationByUuid( organizationUUID ).getName() + "/exporters" )){
+ continue;
+ }
+
+ appFileName = prepareOutputFileName( "application", application.getValue(), null );
+
+
+ EntityManager em = emf.getEntityManager( application.getKey() );
+
+ jg.writeStartArray();
+
+ Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+ long starting_time = System.currentTimeMillis();
+
+ // Loop through the collections. This is the only way to loop
+ // through the entities in the application (former namespace).
+ //could support queries, just need to implement that in the rest endpoint.
+ for ( String collectionName : metadata.keySet() ) {
+ if ( collectionName.equals( "exports" ) ) {
+ continue;
+ }
+ //if the collection you are looping through doesn't match the name of the one you want. Don't export it.
+
+ if ( ( config.get( "collectionName" ) == null ) || collectionName
+ .equals( config.get( "collectionName" ) ) ) {
+ //Query entity manager for the entities in a collection
+ Query query = new Query();
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+ Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
+
+ //pages through the query and backs up all results.
+ PagingResultsIterator itr = new PagingResultsIterator( entities );
+ for ( Object e : itr ) {
+ starting_time = checkTimeDelta( starting_time, jobExecution );
+ Entity entity = ( Entity ) e;
+ jg.writeStartObject();
+ jg.writeFieldName( "Metadata" );
+ jg.writeObject( entity );
+ saveCollectionMembers( jg, em, ( String ) config.get( "collectionName" ), entity );
+ jg.writeEndObject();
+ }
+ }
+ }
+ //}
+
+
+
+ // Close writer and file for this application.
+ jg.writeEndArray();
+
+ jg.close();
+ baos.flush();
+ baos.close();
+
+ //sets up the Inputstream for copying the method to s3.
+ InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+ try {
+ s3Export.copyToS3( is, config, appFileName );
+ }
+ catch ( Exception e ) {
+ export.setState( Export.State.FAILED );
+ return;
+ }
+ }
+ }
+
+
+ /**
+ * Exports a specific applications from an organization
+ */
+ private void exportApplicationFromOrg( UUID organizationUUID, UUID applicationId, final Map<String, Object> config,
final JobExecution jobExecution ) throws Exception {
//retrieves export entity
@@ -328,9 +431,13 @@ public class ExportServiceImpl implements ExportService {
}
+ /**
+ * Exports a specific collection from an org-app combo.
+ */
//might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
- private void exportApplicationForOrg( UUID organizationUUID, UUID applicationUUID, final Map<String, Object> config,
- final JobExecution jobExecution ) throws Exception {
+ private void exportCollectionFromOrgApp( UUID organizationUUID, UUID applicationUUID,
+ final Map<String, Object> config, final JobExecution jobExecution )
+ throws Exception {
//retrieves export entity
UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/844fd5d6/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index f1dd4bb..fa8dd0c 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -1589,40 +1589,43 @@ public class ManagementServiceIT {
OrganizationInfo orgMade = null;
ApplicationInfo appMade = null;
- for(int i = 0; i < 100; i++) {
- orgMade =setup.getMgmtSvc().createOrganization( "minorboss"+i,adminUser,true );
- appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp"+i);
-
- EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
- customMaker.createApplicationCollection( "superappCol"+i );
- //intialize user object to be posted
- Map<String, Object> entityLevelProperties = null;
- Entity[] entNotCopied;
- entNotCopied = new Entity[20];
- //creates entities
- for ( int index = 0; index < 20; index++ ) {
- entityLevelProperties = new LinkedHashMap<String, Object>();
- entityLevelProperties.put( "username", "bobso" + index );
- entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
- entNotCopied[index] = customMaker.create( "superappCol", entityLevelProperties );
+ for ( int i = 0; i < 5; i++ ) {
+ orgMade = setup.getMgmtSvc().createOrganization( "minorboss" + i, adminUser, true );
+ for ( int j = 0; j < 5; j++ ) {
+ appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + j );
+
+ EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
+ customMaker.createApplicationCollection( "superappCol" + j );
+ //intialize user object to be posted
+ Map<String, Object> entityLevelProperties = null;
+ Entity[] entNotCopied;
+ entNotCopied = new Entity[1];
+ //creates entities
+ for ( int index = 0; index < 1; index++ ) {
+ entityLevelProperties = new LinkedHashMap<String, Object>();
+ entityLevelProperties.put( "derp","bacon" );
+// entityLevelProperties.put( "username", "bobso" + index );
+// entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
+ entNotCopied[index] = customMaker.create( "superappCol"+j, entityLevelProperties );
+ }
}
}
payload.put("organizationId",orgMade.getUuid());
- EntityManager em = setup.getEmf().getEntityManager( appMade.getId() );
+ //EntityManager em = setup.getEmf().getEntityManager( appMade.getId() );
//intialize user object to be posted
- Map<String, Object> userProperties = null;
- Entity[] entity;
- entity = new Entity[100];
- //creates entities
- for ( int i = 0; i < 100; i++ ) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "bido" + i );
- userProperties.put( "email", "bido" + i + "@anuff.com" );
-
- entity[i] = em.create( "user", userProperties );
- }
+// Map<String, Object> userProperties = null;
+// Entity[] entity;
+// entity = new Entity[100];
+// //creates entities
+// for ( int i = 0; i < 100; i++ ) {
+// userProperties = new LinkedHashMap<String, Object>();
+// userProperties.put( "username", "bido" + i );
+// userProperties.put( "email", "bido" + i + "@anuff.com" );
+//
+// entity[i] = em.create( "user", userProperties );
+// }
UUID exportUUID = exportService.schedule( payload );
assertNotNull( exportUUID );
@@ -1637,8 +1640,15 @@ public class ManagementServiceIT {
JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
+ //the exporters app is never returned, and I'm not even sure if that is the way we want to go
+ // I feel like there should be a better way than having to store data for the user.
exportService.doExport( jobExecution );
- while (!exportService.getState( appMade.getId(),exportUUID ).equals("FINISHED"));
+ //BiMap<UUID,String> mapper = setup.getMgmtSvc().getApplicationsForOrganization(orgMade.getUuid());
+ //mapper.get( "" )
+// while (!exportService.getState( ,exportUUID )
+// .equals("FINISHED"));
+
+ Thread.sleep( 3000 );
String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( "accessKey" );
@@ -1662,17 +1672,24 @@ public class ManagementServiceIT {
blobStore = context.getBlobStore();
- if(!blobStore.blobExists( bucketName,s3Export.getFilename() )){
- blobStore.deleteContainer( bucketName );
- assert(false);
- }
+// if(!blobStore.blobExists( bucketName,s3Export.getFilename() )){
+// blobStore.deleteContainer( bucketName );
+// assert(false);
+// }
+ //Grab Number of files
Long numOfFiles = blobStore.countBlobs( bucketName );
- Long numWeWant = Long.valueOf( 100 );
- assertEquals( numOfFiles, numWeWant );
-
+ //delete container containing said files
bo = blobStore.getBlob( bucketName,s3Export.getFilename() );
+ blobStore.deleteContainer( bucketName );
+ Long numWeWant = Long.valueOf( 5 );
+ //asserts that the correct number of files was transferred over
+ assertEquals( numWeWant, numOfFiles);
+
+
}catch(Exception e) {
+ blobStore.deleteContainer( bucketName );
+ e.printStackTrace();
assert(false);
}