You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/10/26 20:35:28 UTC
[5/5] usergrid git commit: Fixed using the wrong query package. Added
schedule method that can send verified objects through. First test of changed
flow.
Fixed using the wrong query package.
Added schedule method that can send verified objects through.
First test of changed flow.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/de6d5848
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/de6d5848
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/de6d5848
Branch: refs/heads/export-feature
Commit: de6d584809785576cf59cde5a83f05113fa6e92d
Parents: 78fd382
Author: George Reyes <gr...@apache.org>
Authored: Mon Oct 26 12:35:14 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Mon Oct 26 12:35:14 2015 -0700
----------------------------------------------------------------------
.../organizations/OrganizationResource.java | 7 +-
.../management/export/ExportFilter.java | 2 +-
.../management/export/ExportService.java | 2 +-
.../management/export/ExportServiceImpl.java | 280 +++++++++++--------
.../usergrid/management/export/export_v2.md | 5 +
5 files changed, 180 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index a8cdfa8..789cb56 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -51,7 +51,7 @@ import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.export.ExportFilter;
import org.apache.usergrid.management.export.ExportFilterImpl;
import org.apache.usergrid.management.export.ExportService;
-import org.apache.usergrid.mq.Query;
+import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentials;
import org.apache.usergrid.rest.AbstractContextResource;
@@ -369,10 +369,9 @@ public class OrganizationResource extends AbstractContextResource {
exportFilter.setConnections( connectionSet );
//this references core, there needs to be a better exposed way to do this
//as well as a way to verify queries.
- exportFilter.setQuery(Query.fromQL( query ));
-
-
+ exportFilter.setQuery( Query.fromQL( query ));
+ return exportFilter;
}
@GET
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
index aff5f26..86fdf0b 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
@@ -19,7 +19,7 @@ package org.apache.usergrid.management.export;
import java.util.Set;
-import org.apache.usergrid.mq.Query;
+import org.apache.usergrid.persistence.Query;
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
index db6b105..045fa7f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -37,7 +37,7 @@ public interface ExportService {
* Schedules the export to execute using Objects that represent validated
* credentials and filters.
*/
- UUID schedule (Map<String,Object> json, ExportFilter exportFilter );
+ UUID schedule (Map<String,Object> json, ExportFilter exportFilter ) throws Exception;
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 11b5ae7..4835b63 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -311,65 +311,12 @@ public class ExportServiceImpl implements ExportService {
return;
}
-
//extracts the filter information
ExportFilter exportFilter = parseFilterInformation(jobData);
-
//Start the beginning of the flow.
exportApplicationsFromOrg( (UUID)config.get( "organizationId" ),config,jobExecution,s3Export,exportFilter );
- //we no longer have a concept of an application id. Just the filters from here on in.
-// else if ( config.get( "applicationId" ) == null ) {
-// //exports All the applications from an organization
-// try {
-// logger.debug( "starting export of all application from the following org uuid: "+config.get( "organizationId" ).toString() );
-// exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution, s3Export );
-// }
-// catch ( Exception e ) {
-// export.setErrorMessage( e.getMessage() );
-// export.setState( Export.State.FAILED );
-// em.update( export );
-// return;
-// }
-// }
-// else if ( config.get( "collectionName" ) == null ) {
-// //exports an Application from a single organization
-// try {
-// logger.debug( "Starting export of application: "+ config.get( "applicationId" ).toString());
-// exportApplicationFromOrg( ( UUID ) config.get( "organizationId" ),
-// ( UUID ) config.get( "applicationId" ), config, jobExecution, s3Export );
-// }
-// catch ( Exception e ) {
-// export.setErrorMessage( e.getMessage() );
-// export.setState( Export.State.FAILED );
-// em.update( export );
-// return;
-// }
-// }
-// else {
-// try {
-// //exports a single collection from an app org combo
-// try {
-// logger.debug( "Starting export of the following application collection: "+ config.get( "collectionName" ));
-// exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ), config, jobExecution,
-// s3Export );
-// }
-// catch ( Exception e ) {
-// export.setErrorMessage( e.getMessage() );
-// export.setState( Export.State.FAILED );
-// em.update( export );
-// return;
-// }
-// }
-// catch ( Exception e ) {
-// //if for any reason the backing up fails, then update the entity with a failed state.
-// export.setErrorMessage( e.getMessage() );
-// export.setState( Export.State.FAILED );
-// em.update( export );
-// return;
-// }
-// }
logger.debug( "finished the export job." );
updateExportStatus( export,Export.State.FINISHED,null );
}
@@ -476,45 +423,6 @@ public class ExportServiceImpl implements ExportService {
}
}
-
- /**
- * Exports a specific applications from an organization
- */
- private void exportApplicationFromOrg( UUID organizationUUID, UUID applicationId, final Map<String, Object> config,
- final JobExecution jobExecution, S3Export s3Export ) throws Exception {
-
- //retrieves export entity
- Export export = getExportEntity( jobExecution );
-
- ApplicationInfo application = managementService.getApplicationInfo( applicationId );
- String appFileName = prepareOutputFileName( application.getName(), null );
-
- Map ephemeral = collectionExportAndQuery( applicationId, config, export, jobExecution );
-
- fileTransfer( export, appFileName, ephemeral, config, s3Export );
- }
-
-
- /**
- * Exports a specific collection from an org-app combo.
- */
- //might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
- private void exportCollectionFromOrgApp( UUID applicationUUID, final Map<String, Object> config,
- final JobExecution jobExecution, S3Export s3Export ) throws Exception {
-
- //retrieves export entity
- Export export = getExportEntity( jobExecution );
- ApplicationInfo application = managementService.getApplicationInfo( applicationUUID );
-
- String appFileName = prepareOutputFileName( application.getName(), ( String ) config.get( "collectionName" ) );
-
-
- Map ephemeral = collectionExportAndQuery( applicationUUID, config, export, jobExecution );
-
- fileTransfer( export, appFileName, ephemeral, config, s3Export );
- }
-
-
/**
* Regulates how long to wait until the next heartbeat.
*/
@@ -706,11 +614,10 @@ public class ExportServiceImpl implements ExportService {
}
- protected JsonGenerator getJsonGenerator( File ephermal ) throws IOException {
+ protected JsonGenerator getJsonGenerator( OutputStream entityOutputStream ) throws IOException {
//TODO:shouldn't the below be UTF-16?
- FileOutputStream fileOutputStream = new FileOutputStream( ephermal );
- OutputStream entityOutputStream = new BufferedOutputStream( fileOutputStream );
+ //most systems are little endian.
JsonGenerator jg = jsonFactory.createGenerator( entityOutputStream, JsonEncoding.UTF16_LE );
jg.setPrettyPrinter( new MinimalPrettyPrinter( "" ) );
jg.setCodec( new ObjectMapper() );
@@ -736,6 +643,8 @@ public class ExportServiceImpl implements ExportService {
return outputFileName;
}
+//TODO: GREY need to create a method that will be called with a filename string then generate a output stream.
+ //That output stream will then be fed into the json generator. and added to the entitiesToExport array.
/**
* handles the query and export of collections
@@ -750,20 +659,26 @@ public class ExportServiceImpl implements ExportService {
int entitiesExportedCount = 0;
- //Could easily be converted to take in input streams. Just a harder refactor.
+ //TODO:GREY Add config to change path where this file is written.
+
List<File> entitiesToExport = new ArrayList<>( );
- //TODO:Add config to change path where this file is written.
File entityFileToBeExported = new File( "tempEntityExportPart1");
- JsonGenerator jg = getJsonGenerator( entityFileToBeExported );
entityFileToBeExported.deleteOnExit();
+ FileOutputStream fileEntityOutputStream = new FileOutputStream( entityFileToBeExported );
+ OutputStream entityOutputStream = new BufferedOutputStream( fileEntityOutputStream );
entitiesToExport.add( entityFileToBeExported );
+ JsonGenerator jg = getJsonGenerator( entityOutputStream );
+
+ //While this is more wordy it allows great seperation.
List<File> connectionsToExport = new ArrayList<>( );
File connectionFileToBeExported = new File ("tempConnectionExportPart1");
connectionFileToBeExported.deleteOnExit();
- JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
+ FileOutputStream fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported );
+ OutputStream connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream );
connectionsToExport.add( connectionFileToBeExported );
+ JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionOutputStream );
if(exportFilter.getCollections() == null || exportFilter.getCollections().isEmpty()) {
@@ -777,12 +692,12 @@ public class ExportServiceImpl implements ExportService {
//Query entity manager for the entities in a collection
Query query = null;
- if ( config.get( "query" ) == null ) {
+ if(exportFilter.getQuery() == null ) {
query = new Query();
}
else {
try {
- query = Query.fromQL( ( String ) config.get( "query" ) );
+ query = exportFilter.getQuery();
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -810,10 +725,11 @@ public class ExportServiceImpl implements ExportService {
if ( entitiesExportedCount % 1000 == 0 ) {
//Time to split files
currentFilePartIndex++;
- entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex );
- jg = getJsonGenerator( entityFileToBeExported );
- entityFileToBeExported.deleteOnExit();
- entitiesToExport.add( entityFileToBeExported );
+ File entityFileToBeExported2 = new File( "tempEntityExportPart" + currentFilePartIndex );
+ //TODO: UNCOMMENT THIS OR THE JSON GENERATION WON'T WORK.
+ //jg = getJsonGenerator( entityFileToBeExported2 );
+ entityFileToBeExported2.deleteOnExit();
+ entitiesToExport.add( entityFileToBeExported2 );
//It is quite likely that there are files that do not contain any connections and thus there will not
@@ -821,7 +737,7 @@ public class ExportServiceImpl implements ExportService {
// about that at this point.
connectionFileToBeExported = new File( "tempConnectionExportPart" + currentFilePartIndex );
connectionFileToBeExported.deleteOnExit();
- connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
+ //connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
connectionsToExport.add( connectionFileToBeExported );
}
}
@@ -836,12 +752,12 @@ public class ExportServiceImpl implements ExportService {
for( String collectionName : collectionSet ) {
//Query entity manager for the entities in a collection
Query query = null;
- if ( config.get( "query" ) == null ) {
+ if(exportFilter.getQuery() == null ) {
query = new Query();
}
else {
try {
- query = Query.fromQL( ( String ) config.get( "query" ) );
+ query = exportFilter.getQuery();
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -870,10 +786,14 @@ public class ExportServiceImpl implements ExportService {
//Time to split files
currentFilePartIndex++;
entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex );
- jg = getJsonGenerator( entityFileToBeExported );
entityFileToBeExported.deleteOnExit();
+ fileEntityOutputStream = new FileOutputStream( entityFileToBeExported );
+ entityOutputStream = new BufferedOutputStream( fileEntityOutputStream );
entitiesToExport.add( entityFileToBeExported );
+ jg = getJsonGenerator( entityOutputStream );
+
+
//It is quite likely that there are files that do not contain any connections and thus there will not
@@ -881,7 +801,9 @@ public class ExportServiceImpl implements ExportService {
// about that at this point.
connectionFileToBeExported = new File( "tempConnectionExportPart" + currentFilePartIndex );
connectionFileToBeExported.deleteOnExit();
- connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
+ fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported );
+ connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream );
+ connectionJsonGeneration = getJsonGenerator( connectionOutputStream );
connectionsToExport.add( connectionFileToBeExported );
}
}
@@ -897,6 +819,144 @@ public class ExportServiceImpl implements ExportService {
return filePointers;
}
+ //TODO: GREY need to create a method that will be called with a filename string then generate a output stream.
+ //That output stream will then be fed into the json generator. and added to the entitiesToExport array.
+
+ /**
+ * handles the query and export of collections
+ */
+ //TODO:Needs further refactoring.
+ protected Map collectionExportAndQuery2( UUID applicationUUID, final Map<String, Object> config, Export export,
+ final JobExecution jobExecution,ExportFilter exportFilter ) throws Exception {
+
+ EntityManager em = emf.getEntityManager( applicationUUID );
+ long starting_time = System.currentTimeMillis();
+ //The counter needs to be constant across collections since application exports do across collection aggregation.
+ int entitiesExportedCount = 0;
+
+
+ //TODO:GREY Add config to change path where this file is written.
+
+ List<File> entitiesToExport = new ArrayList<>( );
+ File entityFileToBeExported = new File( "tempEntityExportPart1");
+ entityFileToBeExported.deleteOnExit();
+ FileOutputStream fileEntityOutputStream = new FileOutputStream( entityFileToBeExported );
+ OutputStream entityOutputStream = new BufferedOutputStream( fileEntityOutputStream );
+ entitiesToExport.add( entityFileToBeExported );
+
+ JsonGenerator jg = getJsonGenerator( entityOutputStream );
+
+ //While this is more wordy it allows great seperation.
+ List<File> connectionsToExport = new ArrayList<>( );
+ File connectionFileToBeExported = new File ("tempConnectionExportPart1");
+ connectionFileToBeExported.deleteOnExit();
+ FileOutputStream fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported );
+ OutputStream connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream );
+ connectionsToExport.add( connectionFileToBeExported );
+
+ JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionOutputStream );
+
+
+ if(exportFilter.getCollections() == null || exportFilter.getCollections().isEmpty()) {
+ Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+
+ exportCollection( export, jobExecution, exportFilter, em, starting_time, entitiesExportedCount,
+ entitiesToExport, jg, connectionsToExport, connectionJsonGeneration, metadata.keySet() );
+ }
+ //handles the case where there are specific collections that need to be exported.
+ else{
+ Set<String> collectionSet = exportFilter.getCollections();
+
+ //loop through only the collection names
+ jg = exportCollection( export, jobExecution, exportFilter, em, starting_time, entitiesExportedCount,
+ entitiesToExport, jg, connectionsToExport, connectionJsonGeneration, collectionSet );
+ }
+ jg.flush();
+ jg.close();
+
+ HashMap<String,List> filePointers = new HashMap<>( );
+ filePointers.put( "entities",entitiesToExport );
+ filePointers.put( "connections",connectionsToExport );
+
+ return filePointers;
+ }
+
+
+ private JsonGenerator exportCollection( final Export export, final JobExecution jobExecution,
+ final ExportFilter exportFilter, final EntityManager em, long starting_time,
+ int entitiesExportedCount, final List<File> entitiesToExport,
+ JsonGenerator jg, final List<File> connectionsToExport,
+ JsonGenerator connectionJsonGeneration, final Set<String> collectionSet )
+ throws Exception {
+ final File entityFileToBeExported;
+ final FileOutputStream fileEntityOutputStream;
+ final OutputStream entityOutputStream;
+ final File connectionFileToBeExported;
+ final FileOutputStream fileConnectionOutputStream;
+ final OutputStream connectionOutputStream;
+ for( String collectionName : collectionSet ) {
+ //Query entity manager for the entities in a collection
+ Query query = null;
+ if(exportFilter.getQuery() == null ) {
+ query = new Query();
+ }
+ else {
+ try {
+ query = exportFilter.getQuery();
+ }
+ catch ( Exception e ) {
+ export.setErrorMessage( e.getMessage() );
+ }
+ }
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Level.ALL_PROPERTIES );
+ query.setCollection( collectionName );
+
+ Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
+
+ PagingResultsIterator itr = new PagingResultsIterator( entities );
+ //counter that will inform when we should split into another file.
+ int currentFilePartIndex = 1;
+
+ for ( Object e : itr ) {
+ starting_time = checkTimeDelta( starting_time, jobExecution );
+ Entity entity = ( Entity ) e;
+ jg.writeObject( entity );
+ saveCollectionMembers( jg, em, collectionName, entity );
+ saveConnections( entity, em, connectionJsonGeneration,exportFilter );
+ jg.writeRaw( '\n' );
+ jg.flush();
+ entitiesExportedCount++;
+ if ( entitiesExportedCount % 1000 == 0 ) {
+ //Time to split files
+ currentFilePartIndex++;
+ entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex );
+ entityFileToBeExported.deleteOnExit();
+ fileEntityOutputStream = new FileOutputStream( entityFileToBeExported );
+ entityOutputStream = new BufferedOutputStream( fileEntityOutputStream );
+ entitiesToExport.add( entityFileToBeExported );
+
+ jg = getJsonGenerator( entityOutputStream );
+
+
+ //It is quite likely that there are files that do not contain any connections and thus there will not
+
+
+ //be anything to write to these empty connection files. Not entirely sure what to do
+ // about that at this point.
+ connectionFileToBeExported = new File( "tempConnectionExportPart" + currentFilePartIndex );
+ connectionFileToBeExported.deleteOnExit();
+ fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported );
+ connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream );
+ connectionJsonGeneration = getJsonGenerator( connectionOutputStream );
+ connectionsToExport.add( connectionFileToBeExported );
+ }
+ }
+ }
+ return jg;
+ }
+
+
private S3Export s3ExportDeterminator(final JobData jobData){
Object s3PlaceHolder = jobData.getProperty( "s3Export" );
S3Export s3Export = null;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
index ca86175..387d88a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
@@ -170,6 +170,11 @@ These two lines take an application and search for a specific collection within
If the query is empty/null (need to apply the fix where queries can be null, currently they are always initialized) or if the query is "select *" then we go and search through the graph database for each entity. If the query is more sophisticated then we query elasticsearch for all relevant entity information. (Maybe a useful feature would be to perform a reindexing operation on each collection we wish to export. ) This would work exactly the same way a generic query performed at the rest tier would access the data.
+###What flow does it go through?
+Well first it aggregates a bidirection map of uuid's and application names. Then checks to see if there is any filter being applied. If not then we will export all the applications. If there is a filter then we only check each application that is present in the filter.
+
+Next we go to collections and check if the filter has listed any collection names to be exported. If there aren't, then we export every single collection. If there is a filter then we export and search for all the entities in the collection specific collections that are listed.
+