You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/10 22:22:42 UTC
[13/50] [abbrv] git commit: mocked filename output. Still didn’t fix maven testFileConnection test for export. cleaned up code to only look at one org. Should make export a lot more easy to read/debug.
mocked filename output. Still didn’t fix maven testFileConnection test for export.
cleaned up code to only look at one org. Should make export a lot more easy to read/debug.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/152c72d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/152c72d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/152c72d4
Branch: refs/pull/70/head
Commit: 152c72d4432f592a1fcc226ea4ac559e9dbfb56e
Parents: 88a2a48
Author: grey <gr...@apigee.com>
Authored: Thu Feb 27 17:23:03 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Thu Feb 27 17:23:03 2014 -0800
----------------------------------------------------------------------
.../applications/ApplicationResource.java | 2 +
.../apache/usergrid/management/ExportInfo.java | 4 +
.../management/export/ExportServiceImpl.java | 96 +++++++++++++++++---
.../usergrid/management/export/S3Export.java | 4 +
.../management/export/S3ExportImpl.java | 7 ++
.../cassandra/ManagementServiceIT.java | 28 ++++--
.../management/cassandra/MockS3ExportImpl.java | 11 ++-
7 files changed, 130 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index 0e9d8ed..0ddf61b 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -228,8 +228,10 @@ public class ApplicationResource extends AbstractContextResource {
try {
//parse the json into some useful object (the config params)
ExportInfo objEx = new ExportInfo( json );
+ objEx.setOrganizationId( organization.getUuid() );
objEx.setApplicationId( applicationId );
objEx.setCollection( colExport );
+
jobUUID = exportService.schedule( objEx );
uuidRet.put( "jobUUID", jobUUID.toString() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
index b1be504..d3d0097 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
@@ -22,6 +22,7 @@ public class ExportInfo extends TypedEntity {
private String bucket_location;
private UUID applicationId; //TODO: include application when initializing export info
private String collection; //TODO: include collection when initializing export info
+ private UUID organizationId;
public ExportInfo( Map<String, Object> exportData ) {
@@ -40,6 +41,7 @@ public class ExportInfo extends TypedEntity {
return applicationId;
}
+ public UUID getOrganizationId() { return organizationId; }
public String getPath() {
return path;
@@ -74,4 +76,6 @@ public class ExportInfo extends TypedEntity {
public void setApplicationId( UUID appId ) { applicationId = appId;}
public void setCollection ( String colName) { collection = colName; }
+
+ public void setOrganizationId ( UUID orgId ) { organizationId = orgId; }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index b716293..7e5833f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -140,19 +140,15 @@ public class ExportServiceImpl implements ExportService {
export.setState( Export.State.STARTED );
em.update( export );
- //retrieves all the organizations. Loops through them and backs up the appplications for each one
- Map<UUID, String> organizationGet = getOrgs( config );
- for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
- try {
- //exports all the applications for a single organization
- exportApplicationsForOrg( organization, config, jobExecution );
- }
- catch ( Exception e ) {
- //if for any reason the backing up fails, then update the entity with a failed state.
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
- }
+ try {
+ //exports all the applications for a single organization
+ exportApplicationForOrg( config.getOrganizationId(), config, jobExecution );
+ }
+ catch ( Exception e ) {
+ //if for any reason the backing up fails, then update the entity with a failed state.
+ export.setState( Export.State.FAILED );
+ em.update( export );
+ return;
}
export.setState( Export.State.FINISHED );
em.update( export );
@@ -189,7 +185,6 @@ public class ExportServiceImpl implements ExportService {
return organizationNames;
}
-
public SchedulerService getSch() {
return sch;
}
@@ -307,6 +302,79 @@ public class ExportServiceImpl implements ExportService {
}
}
+ //might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
+ private void exportApplicationForOrg( UUID organizationUUID, final ExportInfo config,
+ final JobExecution jobExecution ) throws Exception {
+
+ //retrieves export entity
+ UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
+ EntityManager exportManager = emf.getEntityManager( config.getApplicationId() );
+ Export export = exportManager.get( exportId, Export.class );
+
+ //sets up a output stream for s3 backup.
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ // Loop through the applications per organization
+ BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organizationUUID );
+ for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+
+ logger.info( application.getValue() + " : " + application.getKey() );
+
+ String appFileName = prepareOutputFileName( "application", application.getValue() );
+
+ JsonGenerator jg = getJsonGenerator( baos );
+
+ EntityManager em = emf.getEntityManager( application.getKey() );
+
+ jg.writeStartArray();
+
+ Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+ long starting_time = System.currentTimeMillis();
+
+ // Loop through the collections. This is the only way to loop
+ // through the entities in the application (former namespace).
+ //could support queries, just need to implement that in the rest endpoint.
+ for ( String collectionName : metadata.keySet() ) {
+ //if the collection you are looping through doesn't match the name of the one you want. Don't export it.
+ if ( collectionName.equals( config.getCollection() ) ) {
+ //Query entity manager for the entities in a collection
+ Query query = new Query();
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+ Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
+
+ //pages through the query and backs up all results.
+ PagingResultsIterator itr = new PagingResultsIterator( entities );
+ for ( Object e : itr ) {
+ starting_time = checkTimeDelta( starting_time, jobExecution );
+ Entity entity = ( Entity ) e;
+ jg.writeStartObject();
+ jg.writeFieldName( "Metadata" );
+ jg.writeObject(entity );
+ saveCollectionMembers( jg, em, config.getCollection(), entity );
+ jg.writeEndObject();
+ }
+ }
+ }
+
+ // Close writer and file for this application.
+ jg.writeEndArray();
+ jg.close();
+ baos.flush();
+ baos.close();
+
+ //sets up the Inputstream for copying the method to s3.
+ InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+ try {
+ s3Export.copyToS3( is, config, appFileName );
+ }
+ catch ( Exception e ) {
+ export.setState( Export.State.FAILED );
+ return;
+ }
+ }
+ }
+
/**
* Regulates how long to wait until the next heartbeat.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
index 941a612..b774db8 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
@@ -13,4 +13,8 @@ import org.apache.usergrid.management.ExportInfo;
public interface S3Export {
void copyToS3( InputStream inputStream, ExportInfo exportInfo, String filename );
+ String getFilename ();
+
+ void setFilename (String givenName);
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 2c274e3..1ca6e97 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -32,6 +32,7 @@ public class S3ExportImpl implements S3Export {
@Override
public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
+
Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
/*won't need any of the properties as I have the export info*/
String bucketName = exportInfo.getBucket_location();
@@ -80,4 +81,10 @@ public class S3ExportImpl implements S3Export {
logger.error( "Error uploading to blob store", e );
}
}
+
+ @Override
+ public String getFilename () {return "";}
+
+ @Override
+ public void setFilename(String givenName) {;}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index b4b82e0..70010d4 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -746,25 +746,29 @@ public class ManagementServiceIT {
//Tests to make sure we can call the job with mock data and it runs.
- @Ignore
+ @Ignore //Connections won't save when run with maven, but on local builds it will.
public void testFileConnections() throws Exception {
File f = null;
try {
- f = new File( "test.json" );
- f.delete();
+ f = new File( "testFileConnections.json" );
+ //f.delete();
}
catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
}
+
S3Export s3Export = new MockS3ExportImpl();
+ s3Export.setFilename( "testFileConnections.json" );
+
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
ExportInfo exportInfo = new ExportInfo( payload );
+ exportInfo.setOrganizationId( organization.getUuid() );
exportInfo.setApplicationId( applicationId );
exportInfo.setCollection( "users" );
@@ -823,7 +827,7 @@ public class ManagementServiceIT {
File f = null;
try {
- f = new File( "test.json" );
+ f = new File( "fileValidity.json" );
f.delete();
}
catch ( Exception e ) {
@@ -831,10 +835,12 @@ public class ManagementServiceIT {
}
S3Export s3Export = new MockS3ExportImpl();
+ s3Export.setFilename( "fileValidity.json" );
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
ExportInfo exportInfo = new ExportInfo( payload );
+ exportInfo.setOrganizationId( organization.getUuid() );
exportInfo.setApplicationId( applicationId );
exportInfo.setCollection( "users" );
@@ -870,7 +876,7 @@ public class ManagementServiceIT {
try {
- f = new File( "test.json" );
+ f = new File( "exportOneOrg.json" );
f.delete();
}
catch ( Exception e ) {
@@ -881,10 +887,12 @@ public class ManagementServiceIT {
"123456789" );
S3Export s3Export = new MockS3ExportImpl();
+ s3Export.setFilename("exportOneOrg.json");
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
ExportInfo exportInfo = new ExportInfo( payload );
+ exportInfo.setOrganizationId( organization.getUuid() );
exportInfo.setApplicationId( applicationId );
exportInfo.setCollection( "roles" );
@@ -925,7 +933,7 @@ public class ManagementServiceIT {
String appName = "testAppNotExported";
try {
- f = new File( "test.json" );
+ f = new File( "exportOneApp.json" );
f.delete();
}
catch ( Exception e ) {
@@ -949,10 +957,12 @@ public class ManagementServiceIT {
}
S3Export s3Export = new MockS3ExportImpl();
+ s3Export.setFilename( "exportOneApp.json" );
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
ExportInfo exportInfo = new ExportInfo( payload );
+ exportInfo.setOrganizationId( organization.getUuid() );
exportInfo.setApplicationId( applicationId );
exportInfo.setCollection( "roles" );
@@ -992,7 +1002,7 @@ public class ManagementServiceIT {
// String appName = "ed-app";
try {
- f = new File( "test.json" );
+ f = new File( "exportOneCollection.json" );
f.delete();
}
catch ( Exception e ) {
@@ -1018,10 +1028,12 @@ public class ManagementServiceIT {
}
S3Export s3Export = new MockS3ExportImpl();
+ s3Export.setFilename( "exportOneCollection.json" );
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
ExportInfo exportInfo = new ExportInfo( payload );
+ exportInfo.setOrganizationId( organization.getUuid() );
exportInfo.setApplicationId( applicationId );
exportInfo.setCollection( "baconators" );
@@ -1061,6 +1073,7 @@ public class ManagementServiceIT {
HashMap<String, Object> payload = payloadBuilder();
ExportInfo exportInfo = new ExportInfo( payload );
+ exportInfo.setOrganizationId( organization.getUuid() );
exportInfo.setApplicationId( applicationId );
@@ -1095,6 +1108,7 @@ public class ManagementServiceIT {
JobExecution jobExecution = mock( JobExecution.class );
ExportInfo exportInfo = new ExportInfo( payload );
+ exportInfo.setOrganizationId( organization.getUuid() );
exportInfo.setApplicationId( applicationId );
UUID entityExportUUID = eS.schedule( exportInfo );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/152c72d4/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index d8a8b08..6419719 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -19,6 +19,7 @@ import org.apache.usergrid.management.export.S3Export;
* Streams / reads the information written from the export service to a file named "test.json"
*/
public class MockS3ExportImpl implements S3Export {
+ public static String filename;
@Override
public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
@@ -28,7 +29,7 @@ public class MockS3ExportImpl implements S3Export {
//FileInputStream fis = new PrintWriter( inputStream );
try {
- outputStream = new FileOutputStream( new File( "test.json" ) );
+ outputStream = new FileOutputStream( new File( getFilename() ) );
}
catch ( FileNotFoundException e ) {
e.printStackTrace();
@@ -44,4 +45,12 @@ public class MockS3ExportImpl implements S3Export {
e.printStackTrace();
}
}
+
+ @Override
+ public String getFilename () {
+ return filename;
+ }
+
+ @Override
+ public void setFilename (String givenName) { filename = givenName; }
}