You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/10/26 20:35:24 UTC
[1/5] usergrid git commit: Heavy changes to how tests are going to be
made and created in the future. Still trying to find out how to automate
verification.
Repository: usergrid
Updated Branches:
refs/heads/export-feature ade6e7881 -> de6d58480
Heavy changes to how tests are going to be made and created in the future. Still trying to find out how to automate verification.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bce8d21c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bce8d21c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bce8d21c
Branch: refs/heads/export-feature
Commit: bce8d21c82bcf60fb93020d26fd4c1ac4497c8b6
Parents: ade6e78
Author: George Reyes <gr...@apache.org>
Authored: Tue Oct 20 16:47:27 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Tue Oct 20 16:47:27 2015 -0700
----------------------------------------------------------------------
.../management/export/MockS3ExportFilterIT.java | 164 ++++++++++++-------
1 file changed, 101 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bce8d21c/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
index 3fe537d..667a36a 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportFilterIT.java
@@ -18,28 +18,35 @@ package org.apache.usergrid.management.export;
import java.io.File;
+import java.io.FileFilter;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
+
import org.apache.usergrid.NewOrgAppAdminRule;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.batch.service.JobSchedulerService;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.mq.Query;
+import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.SimpleEntityRef;
@@ -53,6 +60,8 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.Service;
+import it.unimi.dsi.fastutil.Hash;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -76,6 +85,8 @@ public class MockS3ExportFilterIT extends AbstractServiceIT {
private UserInfo adminUser;
private OrganizationInfo organization;
private UUID applicationId;
+ private ExportService exportService = setup.getExportService();
+
@Before
public void setup() throws Exception {
@@ -110,73 +121,61 @@ public class MockS3ExportFilterIT extends AbstractServiceIT {
applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
}
+ @After
+ public void after() {
+ File dir = new File(".");
+ FileFilter fileFilter = new WildcardFileFilter("entities*.json");
+ File[] files = dir.listFiles(fileFilter);
+ for (File file : files) {
+ file.delete();
+ }
+
+ fileFilter = new WildcardFileFilter("connections*.json");
+ files = dir.listFiles(fileFilter);
+ for (File file : files) {
+ file.delete();
+ }
+
+ }
@Test //Connections won't save when run with maven, but on local builds it will.
public void test1000ConnectionsToSingleEntity() throws Exception {
+ //setup
String testFileName ="testConnectionsOnApplicationEndpoint.json";
-
S3Export s3Export = new MockS3ExportImpl( testFileName );
-
- ExportService exportService = setup.getExportService();
-
String appName = newOrgAppAdminRule.getApplicationInfo().getName();
- HashMap<String, Object> payload = payloadBuilder( appName );
-
- payload.put( "organizationId", organization.getUuid() );
- payload.put( "applicationId", applicationId );
-
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
// intialize user object to be posted
- Map<String, Object> userProperties = null;
- Entity[] entity;
int numberOfEntitiesToBeWritten = 997;
- entity = new Entity[numberOfEntitiesToBeWritten];
-
- // creates entities
- for ( int i = 0; i < numberOfEntitiesToBeWritten; i++ ) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test" + i + "@anuff.com" );
- entity[i] = em.create( "users", userProperties );
- }
+ Entity[] entities = createEntities(numberOfEntitiesToBeWritten);
for(int i = 1; i<numberOfEntitiesToBeWritten; i++){
- em.createConnection( em.get( new SimpleEntityRef( "user", entity[0].getUuid() ) ), "testConnections",
- em.get( new SimpleEntityRef( "user", entity[i].getUuid() ) ) );
+ createConnectionsBetweenEntities( entities[0],entities[i] );
}
- setup.getEntityIndex().refresh( applicationId );
-
- Thread.sleep( 1000 );
+// Create Payload to be sent to export job
+ Set applicationsToBeExported = new HashSet<>( );
+ applicationsToBeExported.add( appName );
+ HashMap<String, Object> payload = payloadBuilder( null,applicationsToBeExported,null,null );
- UUID exportUUID = exportService.schedule( payload );
-
- //create and initialize jobData returned in JobExecution.
- JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
-
- JobExecution jobExecution = mock( JobExecution.class );
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- exportService.doExport( jobExecution );
+ //Starts export. Setups up mocks for the job executor.
+ startExportJob( s3Export, payload );
TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {};
- File exportedFile = new File("entities1"+testFileName);
- exportedFile.delete();
- exportedFile = new File("entities2"+testFileName);
- exportedFile.delete();
+ Set<File> exportedConnectionFiles = returnsEntityFilesExported("connections" ,testFileName );
- final InputStream in = new FileInputStream( "connections1"+testFileName );
+ final InputStream in = new FileInputStream( exportedConnectionFiles.iterator().next() );
try{
ObjectMapper mapper = new ObjectMapper();
JsonParser jp = new JsonFactory( ).createParser( in );
Iterator jsonIterator = mapper.readValues( jp, typeRef);
HashMap jsonEntity = (HashMap)jsonIterator.next();
HashMap entityConnections =
- ( HashMap ) ( jsonEntity ).get( entity[0].getUuid().toString() );
+ ( HashMap ) ( jsonEntity ).get( entities[0].getUuid().toString() );
ArrayList connectionArray = (ArrayList)entityConnections.get( "testconnections" );
+ //verifies that the number of connections should be equal to the number of entities written -1.
assertEquals(numberOfEntitiesToBeWritten-1,connectionArray.size());
}catch(Exception e){
assertTrue(e.getMessage(),false );
@@ -185,14 +184,18 @@ public class MockS3ExportFilterIT extends AbstractServiceIT {
finally{
in.close();
}
+ }
- // Delete the created connection files
- exportedFile = new File("connections1"+testFileName);
- exportedFile.delete();
- exportedFile = new File("connections2"+testFileName);
- exportedFile.delete();
+
+ private void startExportJob( final S3Export s3Export, final HashMap<String, Object> payload ) throws Exception {UUID
+ exportUUID = exportService.schedule( payload );
+ JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+ exportService.doExport( jobExecution );
}
+
public JobData jobDataCreator( HashMap<String, Object> payload, UUID exportUUID, S3Export s3Export ) {
JobData jobData = new JobData();
@@ -204,9 +207,7 @@ public class MockS3ExportFilterIT extends AbstractServiceIT {
return jobData;
}
-
- /*Creates fake payload for testing purposes.*/
- public HashMap<String, Object> payloadBuilder( String orgOrAppName ) {
+ public Map<String, Object> targetBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> target = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
@@ -221,19 +222,27 @@ public class MockS3ExportFilterIT extends AbstractServiceIT {
return payload;
}
- public Map<String, Object> targetBuilder() {
- HashMap<String, Object> payload = new HashMap<String, Object>();
- Map<String, Object> target = new HashMap<String, Object>();
- Map<String, Object> storage_info = new HashMap<String, Object>();
- storage_info.put( "s3_key", "null");
- storage_info.put( "s3_access_id", "null" );
- storage_info.put( "bucket_location", "null" );
+ public Set returnsEntityFilesExported(String filenamePrefix,String filenameSuffix){
- target.put( "storage_provider", "s3" );
- target.put( "storage_info", storage_info );
+ //keep reading files until there aren't any more to read.
+ int index = 1;
+ File exportedFile = new File( filenamePrefix + index + filenameSuffix );
- payload.put( "target", target );
- return payload;
+ Set<File> exportedEntityFiles = new HashSet<>( );
+
+ while(exportedFile.exists()) {
+ exportedEntityFiles.add( exportedFile );
+ index++;
+ exportedFile = new File( filenamePrefix + index + filenameSuffix );
+ }
+
+ return exportedEntityFiles;
+ }
+
+ public void deleteSetOfFiles(Set<File> filesExported){
+ for(File exportedFile : filesExported){
+ exportedFile.delete();
+ }
}
@@ -270,14 +279,43 @@ public class MockS3ExportFilterIT extends AbstractServiceIT {
}
- public Map payloadBuilder(Query query,Set<String> applicationNames,Set<String> collectionNames,
+ public HashMap payloadBuilder(Query query,Set<String> applicationNames,Set<String> collectionNames,
Set<String> connectionNames){
Map target = targetBuilder();
- Map filter = filterBuilder(query,applicationNames,collectionNames,connectionNames);
+ Map filters = filterBuilder(query,applicationNames,collectionNames,connectionNames);
+ HashMap payload = new HashMap<>( );
-
+ payload.put( "target",target );
+ payload.put( "filters", filters );
+ return payload;
+ }
+
+ public Entity[] createEntities(final int numberOfEntitiesToBeCreated) throws Exception{
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
+ // intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[numberOfEntitiesToBeCreated];
+
+ // creates entities
+ for ( int i = 0; i < numberOfEntitiesToBeCreated; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test" + i + "@anuff.com" );
+ entity[i] = em.create( "users", userProperties );
+ }
+ //refresh so entities appear immediately
+ setup.getEntityIndex().refresh( applicationId );
+
+ return entity;
+ }
+
+ public ConnectionRef createConnectionsBetweenEntities (Entity entity1, Entity entity2) throws Exception{
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ return em.createConnection( em.get( new SimpleEntityRef( "user", entity1.getUuid() ) ), "testConnections", em.get( new SimpleEntityRef( "user", entity2.getUuid() ) ) );
}
}
[4/5] usergrid git commit: Applied all the filter logic to the export
service.
Posted by gr...@apache.org.
Applied all the filter logic to the export service.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/78fd3828
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/78fd3828
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/78fd3828
Branch: refs/heads/export-feature
Commit: 78fd3828fd00a2eb156187866ea2e5ea47f090bc
Parents: d1f71e2
Author: George Reyes <gr...@apache.org>
Authored: Fri Oct 23 14:30:56 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Fri Oct 23 14:30:56 2015 -0700
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 184 +++++++++++++++----
1 file changed, 145 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/78fd3828/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 353714a..11b5ae7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -416,7 +416,7 @@ public class ExportServiceImpl implements ExportService {
/**
- * Exports All Applications from an Organization
+ * Exports filtered applications or all of the applications.
*/
private void exportApplicationsFromOrg( UUID organizationUUID, final Map<String, Object> config,
final JobExecution jobExecution, S3Export s3Export, ExportFilter exportFilter ) throws Exception {
@@ -425,20 +425,39 @@ public class ExportServiceImpl implements ExportService {
Export export = getExportEntity( jobExecution );
String appFileName = null;
- BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organizationUUID );
- for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+ //if the application filter is empty then we need to export all of the applications.
+ if(exportFilter.getApplications() == null || exportFilter.getApplications().isEmpty()) {
- if ( application.getValue().equals(
+ BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organizationUUID );
+
+ for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+
+ if ( application.getValue().equals(
managementService.getOrganizationByUuid( organizationUUID ).getName() + "/exports" ) ) {
- continue;
+ continue;
+ }
+
+ appFileName = prepareOutputFileName( application.getValue(), null );
+
+ Map ephemeral = collectionExportAndQuery( application.getKey(), config, export, jobExecution,exportFilter );
+
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
+ }
+ //export filter lists specific applications that need to be exported.
+ else{
+ BiMap<String,UUID> applications = managementService.getApplicationsForOrganization( organizationUUID ).inverse();
+ Set<String> applicationSet = exportFilter.getApplications();
- appFileName = prepareOutputFileName( application.getValue(), null );
- Map ephemeral = collectionExportAndQuery( application.getKey(), config, export, jobExecution );
+ for(String applicationName : applicationSet){
+ appFileName = prepareOutputFileName( applicationName, null );
- fileTransfer( export, appFileName, ephemeral, config, s3Export );
+ Map ephemeral = collectionExportAndQuery(applications.get( applicationName ), config, export, jobExecution,exportFilter );
+
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
+ }
}
}
@@ -630,30 +649,55 @@ public class ExportServiceImpl implements ExportService {
/**
* Persists the connection for this entity.
*/
- private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+ private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg, ExportFilter exportFilter ) throws Exception {
//Short circut if you don't find any connections.
Set<String> connectionTypes = em.getConnectionsAsSource( entity );
- if(connectionTypes.size() == 0)
+ if(connectionTypes.size() == 0){
+ logger.debug( "No connections found for this entity. uuid : "+entity.getUuid().toString());
return;
+ }
- jg.writeStartObject();
- jg.writeFieldName( entity.getUuid().toString() );
- jg.writeStartObject();
+ //filtering gets applied here.
+ if(exportFilter.getConnections() == null || exportFilter.getConnections().isEmpty()) {
+
+ jg.writeStartObject();
+ jg.writeFieldName( entity.getUuid().toString() );
+ jg.writeStartObject();
- for ( String connectionType : connectionTypes ) {
+ for ( String connectionType : connectionTypes ) {
- jg.writeFieldName( connectionType );
- jg.writeStartArray();
+ jg.writeFieldName( connectionType );
+ jg.writeStartArray();
- Results results = em.getTargetEntities( entity,connectionType, null, Level.ALL_PROPERTIES );
- PagingResultsIterator connectionResults = new PagingResultsIterator( results );
- for(Object c : connectionResults){
- Entity connectionRef = (Entity) c;
- jg.writeObject( connectionRef.getUuid());
+ Results results = em.getTargetEntities( entity, connectionType, null, Level.ALL_PROPERTIES );
+ PagingResultsIterator connectionResults = new PagingResultsIterator( results );
+ for ( Object c : connectionResults ) {
+ Entity connectionRef = ( Entity ) c;
+ jg.writeObject( connectionRef.getUuid() );
+ }
+
+ jg.writeEndArray();
}
+ }
+ //filter connections
+ else {
+ Set<String> connectionSet = exportFilter.getConnections();
+ for ( String connectionType : connectionTypes ) {
+ if(connectionSet.contains( connectionType )) {
+ jg.writeFieldName( connectionType );
+ jg.writeStartArray();
+
+ Results results = em.getTargetEntities( entity, connectionType, null, Level.ALL_PROPERTIES );
+ PagingResultsIterator connectionResults = new PagingResultsIterator( results );
+ for ( Object c : connectionResults ) {
+ Entity connectionRef = ( Entity ) c;
+ jg.writeObject( connectionRef.getUuid() );
+ }
- jg.writeEndArray();
+ jg.writeEndArray();
+ }
+ }
}
jg.writeEndObject();
jg.writeEndObject();
@@ -698,10 +742,9 @@ public class ExportServiceImpl implements ExportService {
*/
//TODO:Needs further refactoring.
protected Map collectionExportAndQuery( UUID applicationUUID, final Map<String, Object> config, Export export,
- final JobExecution jobExecution ) throws Exception {
+ final JobExecution jobExecution,ExportFilter exportFilter ) throws Exception {
EntityManager em = emf.getEntityManager( applicationUUID );
- Map<String, Object> metadata = em.getApplicationCollectionMetadata();
long starting_time = System.currentTimeMillis();
//The counter needs to be constant across collections since application exports do across collection aggregation.
int entitiesExportedCount = 0;
@@ -721,14 +764,76 @@ public class ExportServiceImpl implements ExportService {
JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
connectionsToExport.add( connectionFileToBeExported );
- for ( String collectionName : metadata.keySet() ) {
- if ( collectionName.equals( "exports" ) ) {
- continue;
+
+ if(exportFilter.getCollections() == null || exportFilter.getCollections().isEmpty()) {
+ Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+ for ( String collectionName : metadata.keySet() ) {
+
+
+ //if the collection you are looping through doesn't match the name of the one you want. Don't export it.
+ if ( ( config.get( "collectionName" ) == null ) || collectionName
+ .equalsIgnoreCase( ( String ) config.get( "collectionName" ) ) ) {
+
+ //Query entity manager for the entities in a collection
+ Query query = null;
+ if ( config.get( "query" ) == null ) {
+ query = new Query();
+ }
+ else {
+ try {
+ query = Query.fromQL( ( String ) config.get( "query" ) );
+ }
+ catch ( Exception e ) {
+ export.setErrorMessage( e.getMessage() );
+ }
+ }
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Level.ALL_PROPERTIES );
+ query.setCollection( collectionName );
+
+ //counter that will inform when we should split into another file.
+ Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
+
+ PagingResultsIterator itr = new PagingResultsIterator( entities );
+ int currentFilePartIndex = 1;
+
+ for ( Object e : itr ) {
+ starting_time = checkTimeDelta( starting_time, jobExecution );
+ Entity entity = ( Entity ) e;
+ jg.writeObject( entity );
+ saveCollectionMembers( jg, em, ( String ) config.get( "collectionName" ), entity );
+ saveConnections( entity, em, connectionJsonGeneration,exportFilter );
+ jg.writeRaw( '\n' );
+ jg.flush();
+ entitiesExportedCount++;
+ if ( entitiesExportedCount % 1000 == 0 ) {
+ //Time to split files
+ currentFilePartIndex++;
+ entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex );
+ jg = getJsonGenerator( entityFileToBeExported );
+ entityFileToBeExported.deleteOnExit();
+ entitiesToExport.add( entityFileToBeExported );
+
+ //It is quite likely that there are files that do not contain any connections and thus there will not
+
+ //be anything to write to these empty connection files. Not entirely sure what to do
+ // about that at this point.
+ connectionFileToBeExported = new File( "tempConnectionExportPart" + currentFilePartIndex );
+ connectionFileToBeExported.deleteOnExit();
+ connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
+ connectionsToExport.add( connectionFileToBeExported );
+ }
+ }
+ }
}
- //if the collection you are looping through doesn't match the name of the one you want. Don't export it.
- if ( ( config.get( "collectionName" ) == null ) || collectionName.equalsIgnoreCase((String)config.get( "collectionName" ) ) ) {
+ }
+ //handles the case where there are specific collections that need to be exported.
+ else{
+ Set<String> collectionSet = exportFilter.getCollections();
+ //loop through only the collection names
+ for( String collectionName : collectionSet ) {
//Query entity manager for the entities in a collection
Query query = null;
if ( config.get( "query" ) == null ) {
@@ -746,40 +851,41 @@ public class ExportServiceImpl implements ExportService {
query.setResultsLevel( Level.ALL_PROPERTIES );
query.setCollection( collectionName );
- //counter that will inform when we should split into another file.
Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
PagingResultsIterator itr = new PagingResultsIterator( entities );
+ //counter that will inform when we should split into another file.
int currentFilePartIndex = 1;
for ( Object e : itr ) {
starting_time = checkTimeDelta( starting_time, jobExecution );
Entity entity = ( Entity ) e;
jg.writeObject( entity );
- saveCollectionMembers( jg, em, ( String ) config.get( "collectionName" ), entity );
- saveConnections( entity, em, connectionJsonGeneration );
+ saveCollectionMembers( jg, em, collectionName, entity );
+ saveConnections( entity, em, connectionJsonGeneration,exportFilter );
jg.writeRaw( '\n' );
jg.flush();
entitiesExportedCount++;
- if(entitiesExportedCount%1000==0){
+ if ( entitiesExportedCount % 1000 == 0 ) {
//Time to split files
currentFilePartIndex++;
- entityFileToBeExported = new File( "tempEntityExportPart"+currentFilePartIndex);
+ entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex );
jg = getJsonGenerator( entityFileToBeExported );
entityFileToBeExported.deleteOnExit();
entitiesToExport.add( entityFileToBeExported );
- //It is quite likely that there are files that do not contain any connections and thus there will not
- //be anything to write to these empty connection files. Not entirely sure what to do about that at this point.
- connectionFileToBeExported = new File ("tempConnectionExportPart"+currentFilePartIndex);
+ //It is quite likely that there are files that do not contain any connections and thus there will not
+
+
+ //be anything to write to these empty connection files. Not entirely sure what to do
+ // about that at this point.
+ connectionFileToBeExported = new File( "tempConnectionExportPart" + currentFilePartIndex );
connectionFileToBeExported.deleteOnExit();
connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
connectionsToExport.add( connectionFileToBeExported );
}
-
}
}
-
}
jg.flush();
jg.close();
[2/5] usergrid git commit: Adding changes for filters
Posted by gr...@apache.org.
Adding changes for filters
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f51b73cf
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f51b73cf
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f51b73cf
Branch: refs/heads/export-feature
Commit: f51b73cfa618fe0a66ad70210bcd5d95b73e066b
Parents: bce8d21
Author: George Reyes <gr...@apache.org>
Authored: Thu Oct 22 13:04:33 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Thu Oct 22 13:04:33 2015 -0700
----------------------------------------------------------------------
.../organizations/OrganizationResource.java | 1 +
.../management/export/ExportFilter.java | 45 ++++
.../management/export/ExportFilterImpl.java | 83 +++++++
.../management/export/ExportServiceImpl.java | 222 ++++++++++++-------
.../usergrid/management/export/export_v2.md | 9 +-
5 files changed, 284 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index afdb8cd..679451a 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -326,6 +326,7 @@ public class OrganizationResource extends AbstractContextResource {
throw new NullArgumentException( "Could not find field 's3_key'" );
}
+ //organizationid is added after the fact so that
json.put( "organizationId",organization.getUuid());
jobUUID = exportService.schedule( json );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
new file mode 100644
index 0000000..aff5f26
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.management.export;
+
+
+import java.util.Set;
+
+import org.apache.usergrid.mq.Query;
+
+
+/**
+ * A model of the filters included in usergrid and how they can be added to.
+ */
+public interface ExportFilter {
+ public Set getApplications();
+
+ public Set getCollections();
+
+ public Set getConnections();
+
+ public Query getQuery();
+
+ public void setApplications(Set applications);
+
+ public void setCollections(Set collections);
+
+ public void setConnections(Set connections);
+
+ public void setQuery(Query query);
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilterImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilterImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilterImpl.java
new file mode 100644
index 0000000..085739e
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilterImpl.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.management.export;
+
+
+import java.util.Set;
+
+import org.apache.usergrid.mq.Query;
+
+
+/**
+ * Implementation that also parses the json data to get the filter information.
+ */
+public class ExportFilterImpl implements ExportFilter {
+ public Query query;
+
+ private Set applications;
+
+ private Set collections;
+
+ private Set connections;
+
+ @Override
+ public Set getApplications() {
+ return applications;
+ }
+
+
+ @Override
+ public Set getCollections() {
+ return collections;
+ }
+
+
+ @Override
+ public Set getConnections() {
+ return connections;
+ }
+
+
+ @Override
+ public Query getQuery() {
+ return query;
+ }
+
+
+ @Override
+ public void setApplications( final Set applications ) {
+ this.applications = applications
+ }
+
+
+ @Override
+ public void setCollections( final Set collections ) {
+ this.collections = collections;
+ }
+
+
+ @Override
+ public void setConnections( final Set connections ) {
+ this.connections = connections;
+ }
+
+
+ @Override
+ public void setQuery( final Query query ) {
+ this.query = query;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 9eb4836..2f63096 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -31,6 +31,8 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import javax.management.RuntimeErrorException;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,6 +74,10 @@ public class ExportServiceImpl implements ExportService {
//injected the Entity Manager Factory to access entity manager
protected EntityManagerFactory emf;
+ //EntityManager that will only be used for the management application and updating export entities
+ protected EntityManager em;
+
+
//inject Management Service to access Organization Data
private ManagementService managementService;
@@ -85,6 +91,10 @@ public class ExportServiceImpl implements ExportService {
private JsonFactory jsonFactory = new JsonFactory();
+ public ExportServiceImpl(){
+ em = emf.getEntityManager( emf.getManagementAppId() );
+ }
+
@Override
public UUID schedule( final Map<String, Object> config ) throws Exception {
@@ -195,101 +205,123 @@ public class ExportServiceImpl implements ExportService {
}
+ //This flow that is detailed
+ //The responsibilities of this method is to error check the configuration that is passed to us in job execution
+ //Then it also delegates to the correct type of export.
+
+ //Seperate into two methods, one for error checking and the other for checking the filters?
+
+ //what should this method do? It should handle the flow of export. Aka by looking at this method we should be able to see
+ //the different steps that we need to take in order to do a export.
+
+ //Extract the job data
+ //Determine
@Override
public void doExport( final JobExecution jobExecution ) throws Exception {
- Map<String, Object> config = ( Map<String, Object> ) jobExecution.getJobData().getProperty( "exportInfo" );
- Object s3PlaceHolder = jobExecution.getJobData().getProperty( "s3Export" );
- S3Export s3Export = null;
+ final JobData jobData = jobExecution.getJobData();
+
+ Map<String, Object> config = ( Map<String, Object> ) jobData.getProperty( "exportInfo" );
if ( config == null ) {
logger.error( "Export Information passed through is null" );
return;
}
- //get the entity manager for the application, and the entity that this Export corresponds to.
- UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
- EntityManager em = emf.getEntityManager( emf.getManagementAppId() );
+ UUID exportId = ( UUID ) jobData.getProperty( EXPORT_ID );
+
+ //TODO:GREY doesn't need to get referenced everytime. Should only be set once and then used everywhere.
+ // EntityManager em = emf.getEntityManager( emf.getManagementAppId() );
Export export = em.get( exportId, Export.class );
//update the entity state to show that the job has officially started.
logger.debug( "Starting export job with uuid: "+export.getUuid() );
export.setState( Export.State.STARTED );
em.update( export );
+
+ //Checks to see if the job was given a different s3 export class. ( Local or Aws )
try {
- if ( s3PlaceHolder != null ) {
- s3Export = ( S3Export ) s3PlaceHolder;
- }
- else {
- s3Export = new AwsS3ExportImpl();
- }
- }
- catch ( Exception e ) {
- logger.error( "S3Export doesn't exist" );
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
+ S3Export s3Export = s3ExportDeterminator( jobData );
+ }catch(Exception e) {
+ updateExportStatus( export, Export.State.FAILED, e.getMessage() );
+ throw e;
}
+
+ //All verification of the job data should be done on the rest tier so at this point we shouldn't need
+ //to error check.
+
+
+
+ //No longer need this specific kind of flow, but what we do need is to check the filters
+ //the filters will tell us how we need to proceed.
+
+
+ //This is defensive programming against anybody who wants to run the export job.
+ //They need to add the organization id or else we won't know where the job came from or what it has
+ //access to.
if ( config.get( "organizationId" ) == null ) {
- logger.error( "No organization could be found" );
- export.setState( Export.State.FAILED );
- em.update( export );
+ logger.error( "No organization uuid was associated with this call. Exiting." );
+ updateExportStatus( export, Export.State.FAILED,"No organization could be found" );
return;
}
- else if ( config.get( "applicationId" ) == null ) {
- //exports All the applications from an organization
- try {
- logger.debug( "starting export of all application from the following org uuid: "+config.get( "organizationId" ).toString() );
- exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution, s3Export );
- }
- catch ( Exception e ) {
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
- }
- }
- else if ( config.get( "collectionName" ) == null ) {
- //exports an Application from a single organization
- try {
- logger.debug( "Starting export of application: "+ config.get( "applicationId" ).toString());
- exportApplicationFromOrg( ( UUID ) config.get( "organizationId" ),
- ( UUID ) config.get( "applicationId" ), config, jobExecution, s3Export );
- }
- catch ( Exception e ) {
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
- }
- }
- else {
- try {
- //exports a single collection from an app org combo
- try {
- logger.debug( "Starting export of the following application collection: "+ config.get( "collectionName" ));
- exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ), config, jobExecution,
- s3Export );
- }
- catch ( Exception e ) {
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
- }
- }
- catch ( Exception e ) {
- //if for any reason the backing up fails, then update the entity with a failed state.
- export.setErrorMessage( e.getMessage() );
- export.setState( Export.State.FAILED );
- em.update( export );
- return;
- }
- }
+
+
+ //extracts the filter information
+ parseFilterInformation(jobData);
+
+ //we no longer have a concept of an application id. Just the filters from here on in.
+// else if ( config.get( "applicationId" ) == null ) {
+// //exports All the applications from an organization
+// try {
+// logger.debug( "starting export of all application from the following org uuid: "+config.get( "organizationId" ).toString() );
+// exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution, s3Export );
+// }
+// catch ( Exception e ) {
+// export.setErrorMessage( e.getMessage() );
+// export.setState( Export.State.FAILED );
+// em.update( export );
+// return;
+// }
+// }
+// else if ( config.get( "collectionName" ) == null ) {
+// //exports an Application from a single organization
+// try {
+// logger.debug( "Starting export of application: "+ config.get( "applicationId" ).toString());
+// exportApplicationFromOrg( ( UUID ) config.get( "organizationId" ),
+// ( UUID ) config.get( "applicationId" ), config, jobExecution, s3Export );
+// }
+// catch ( Exception e ) {
+// export.setErrorMessage( e.getMessage() );
+// export.setState( Export.State.FAILED );
+// em.update( export );
+// return;
+// }
+// }
+// else {
+// try {
+// //exports a single collection from an app org combo
+// try {
+// logger.debug( "Starting export of the following application collection: "+ config.get( "collectionName" ));
+// exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ), config, jobExecution,
+// s3Export );
+// }
+// catch ( Exception e ) {
+// export.setErrorMessage( e.getMessage() );
+// export.setState( Export.State.FAILED );
+// em.update( export );
+// return;
+// }
+// }
+// catch ( Exception e ) {
+// //if for any reason the backing up fails, then update the entity with a failed state.
+// export.setErrorMessage( e.getMessage() );
+// export.setState( Export.State.FAILED );
+// em.update( export );
+// return;
+// }
+// }
logger.debug( "finished the export job." );
- export.setState( Export.State.FINISHED );
- em.update( export );
+ updateExportStatus( export,Export.State.FINISHED,null );
}
@@ -708,4 +740,46 @@ public class ExportServiceImpl implements ExportService {
return filePointers;
}
+
+ private S3Export s3ExportDeterminator(final JobData jobData){
+ Object s3PlaceHolder = jobData.getProperty( "s3Export" );
+ S3Export s3Export = null;
+
+ try {
+ if ( s3PlaceHolder != null ) {
+ s3Export = ( S3Export ) s3PlaceHolder;
+ }
+ else {
+ s3Export = new AwsS3ExportImpl();
+ }
+ }
+ catch ( Exception e ) {
+ logger.error( "S3Export doesn't exist." );
+ throw e;
+ }
+ return s3Export;
+ }
+
+ public void updateExportStatus(Export exportEntity,Export.State exportState,String failureString) throws Exception{
+ if(failureString != null)
+ exportEntity.setErrorMessage( failureString );
+
+ exportEntity.setState( exportState );
+
+ try {
+ em.update( exportEntity );
+ }catch(Exception e){
+ logger.error( "Encountered error updating export entity! " + e.getMessage() );
+ throw e;
+ }
+ }
+
+ //All of this data is vaidated in the rest tier so it can be passed straight through here
+ //TODO: GREY find a way to pass validated object data into the scheduler.
+ public ExportFilter parseFilterInformation(JobData jobData){
+ Map<String,Object> filterData = ( Map<String, Object> ) jobData.getProperty( "filters" );
+ String query = filterData.get( "ql" );
+
+
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f51b73cf/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
index fb9eb5e..ca86175 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
@@ -4,9 +4,11 @@ Defines how the future iterated export handles what gets exported.
##Endpoints
Endpoints are the same as export v1.
+<!--
+Not needed because the filters handle what we want exported and what application it should be looking at. Not the endpoints.
` POST /management/orgs/<org_name>/apps/<app_name>/collection/<collection_name>/export `
-` POST /management/orgs/<org_name>/apps/<app_name>/export `
+` POST /management/orgs/<org_name>/apps/<app_name>/export `-->
` POST /management/orgs/<org_name>/export`
@@ -15,7 +17,7 @@ Endpoints are the same as export v1.
##What payload to the post endpoints take?
- curl -X POST -i -H 'Authorization: Bearer <your admin token goes here>' 'http://localhost:8080/management/orgs/<org_name>/apps/<app_name>/export' -d
+ curl -X POST -i -H 'Authorization: Bearer <your admin token goes here>' 'http://localhost:8080/management/orgs/<org_name>/export' -d
'{"target":{
"storage_provider":"s3",
"storage_info":{
@@ -50,6 +52,9 @@ There are 4 ways that you can filter data out by.
- Connections
- In order to export specific connections you can list them same as the other filters. If this is filled in then you are only exporting the connections that contain the names listed in the connections json array.
- If you want to export all the connections then delete the filter and it will export all the connections.
+
+####What happens if my data is invalid?
+If you try to export data that doesn't exist in your filter then the call will fail and you will get returned a list of the data that is invalid.
##Data Format for export
[3/5] usergrid git commit: Added some logic to support new filter
logic and passing it through Export.
Posted by gr...@apache.org.
Added some logic to support new filter logic and passing it through Export.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d1f71e2b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d1f71e2b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d1f71e2b
Branch: refs/heads/export-feature
Commit: d1f71e2b64c4c57f39f34451d293c4b2df1d6969
Parents: f51b73c
Author: George Reyes <gr...@apache.org>
Authored: Thu Oct 22 16:44:01 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Thu Oct 22 16:44:01 2015 -0700
----------------------------------------------------------------------
.../organizations/OrganizationResource.java | 32 +++++++-
.../management/export/ExportService.java | 6 ++
.../management/export/ExportServiceImpl.java | 84 +++++++++++++++-----
3 files changed, 102 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d1f71e2b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index 679451a..a8cdfa8 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.rest.management.organizations;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import javax.ws.rs.Consumes;
@@ -47,7 +48,10 @@ import org.apache.commons.lang.NullArgumentException;
import org.apache.usergrid.management.ActivationState;
import org.apache.usergrid.management.OrganizationConfig;
import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.export.ExportFilter;
+import org.apache.usergrid.management.export.ExportFilterImpl;
import org.apache.usergrid.management.export.ExportService;
+import org.apache.usergrid.mq.Query;
import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentials;
import org.apache.usergrid.rest.AbstractContextResource;
@@ -298,6 +302,8 @@ public class OrganizationResource extends AbstractContextResource {
Map<String,Object> properties;
Map<String, Object> storage_info;
+ //the storage providers could be an abstract class that others can implement in order to try to pull out
+ //their own data.
try {
if((properties = ( Map<String, Object> ) json.get( "properties" )) == null){
throw new NullArgumentException("Could not find 'properties'");
@@ -326,10 +332,14 @@ public class OrganizationResource extends AbstractContextResource {
throw new NullArgumentException( "Could not find field 's3_key'" );
}
+
+
+
//organizationid is added after the fact so that
json.put( "organizationId",organization.getUuid());
+ ExportFilter exportFilter = exportFilterParser( json );
+ jobUUID = exportService.schedule( json,exportFilter );
- jobUUID = exportService.schedule( json );
uuidRet.put( "Export Entity", jobUUID.toString() );
}
catch ( NullArgumentException e ) {
@@ -345,6 +355,26 @@ public class OrganizationResource extends AbstractContextResource {
return Response.status( SC_ACCEPTED ).entity( uuidRet ).build();
}
+ //need to explore a validate query method.
+ public ExportFilter exportFilterParser(Map<String,Object> input){
+ String query = ( String ) input.get( "ql" );
+ Set applicationSet = ( Set ) input.get( "apps" );
+ Set collectionSet = ( Set ) input.get( "collections" );
+ Set connectionSet = ( Set ) input.get( "connections" );
+
+ //TODO:GREY move export filter to the top of this .
+ ExportFilter exportFilter = new ExportFilterImpl();
+ exportFilter.setApplications( applicationSet );
+ exportFilter.setCollections( collectionSet );
+ exportFilter.setConnections( connectionSet );
+ //this references core, there needs to be a better exposed way to do this
+ //as well as a way to verify queries.
+ exportFilter.setQuery(Query.fromQL( query ));
+
+
+
+ }
+
@GET
@RequireOrganizationAccess
@Path("export/{exportEntity: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}")
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d1f71e2b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
index 5a14012..db6b105 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -33,6 +33,12 @@ public interface ExportService {
*/
UUID schedule( Map<String,Object> json) throws Exception;
+ /**
+ * Schedules the export to execute using Objects that represent validated
+ * credentials and filters.
+ */
+ UUID schedule (Map<String,Object> json, ExportFilter exportFilter );
+
/**
* Perform the export to the external resource
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d1f71e2b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 2f63096..353714a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -150,6 +150,61 @@ public class ExportServiceImpl implements ExportService {
return export.getUuid();
}
+ @Override
+ public UUID schedule( final Map<String, Object> config, final ExportFilter exportFilter ) throws Exception {
+ logger.debug( "Starting to schedule the export job" );
+
+ if ( config == null ) {
+ logger.error( "export information cannot be null" );
+ return null;
+ }
+
+ EntityManager em = null;
+ try {
+ em = emf.getEntityManager( emf.getManagementAppId() );
+ Set<String> collections = em.getApplicationCollections();
+
+ if ( !collections.contains( "exports" ) ) {
+ em.createApplicationCollection( "exports" );
+ }
+ }
+ catch ( Exception e ) {
+ logger.error( "application doesn't exist within the current context" );
+ return null;
+ }
+
+ Export export = new Export();
+
+ //update state
+ try {
+ export = em.create( export );
+ }
+ catch ( Exception e ) {
+ logger.error( "Export entity creation failed" );
+ return null;
+ }
+
+ export.setState( Export.State.CREATED );
+ em.update( export );
+ //set data to be transferred to exportInfo
+ JobData jobData = new JobData();
+ jobData.setProperty( "target", config );
+ jobData.setProperty( "filters", exportFilter);
+ jobData.setProperty( EXPORT_ID, export.getUuid() );
+
+
+ long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
+
+ //schedule job
+ logger.debug( "Creating the export job with the name: "+ EXPORT_JOB_NAME );
+ sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
+
+ //update state
+ updateExportStatus( export, Export.State.SCHEDULED,null );
+
+ return export.getUuid();
+ }
+
/**
* Query Entity Manager for the string state of the Export Entity. This corresponds to the GET /export
@@ -239,23 +294,14 @@ public class ExportServiceImpl implements ExportService {
em.update( export );
//Checks to see if the job was given a different s3 export class. ( Local or Aws )
+ S3Export s3Export = null;
try {
- S3Export s3Export = s3ExportDeterminator( jobData );
+ s3Export = s3ExportDeterminator( jobData );
}catch(Exception e) {
updateExportStatus( export, Export.State.FAILED, e.getMessage() );
throw e;
}
-
- //All verification of the job data should be done on the rest tier so at this point we shouldn't need
- //to error check.
-
-
-
- //No longer need this specific kind of flow, but what we do need is to check the filters
- //the filters will tell us how we need to proceed.
-
-
//This is defensive programming against anybody who wants to run the export job.
//They need to add the organization id or else we won't know where the job came from or what it has
//access to.
@@ -267,7 +313,11 @@ public class ExportServiceImpl implements ExportService {
//extracts the filter information
- parseFilterInformation(jobData);
+ ExportFilter exportFilter = parseFilterInformation(jobData);
+
+
+ //Start the beginning of the flow.
+ exportApplicationsFromOrg( (UUID)config.get( "organizationId" ),config,jobExecution,s3Export,exportFilter );
//we no longer have a concept of an application id. Just the filters from here on in.
// else if ( config.get( "applicationId" ) == null ) {
@@ -369,7 +419,7 @@ public class ExportServiceImpl implements ExportService {
* Exports All Applications from an Organization
*/
private void exportApplicationsFromOrg( UUID organizationUUID, final Map<String, Object> config,
- final JobExecution jobExecution, S3Export s3Export ) throws Exception {
+ final JobExecution jobExecution, S3Export s3Export, ExportFilter exportFilter ) throws Exception {
//retrieves export entity
Export export = getExportEntity( jobExecution );
@@ -774,12 +824,8 @@ public class ExportServiceImpl implements ExportService {
}
}
- //All of this data is vaidated in the rest tier so it can be passed straight through here
- //TODO: GREY find a way to pass validated object data into the scheduler.
public ExportFilter parseFilterInformation(JobData jobData){
- Map<String,Object> filterData = ( Map<String, Object> ) jobData.getProperty( "filters" );
- String query = filterData.get( "ql" );
-
-
+ ExportFilter exportFilter = ( ExportFilter ) jobData.getProperty("filter");
+ return exportFilter;
}
}
[5/5] usergrid git commit: Fixed using the wrong query package. Added
schedule method that can send verified objects through. First test of changed
flow.
Posted by gr...@apache.org.
Fixed using the wrong query package.
Added schedule method that can send verified objects through.
First test of changed flow.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/de6d5848
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/de6d5848
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/de6d5848
Branch: refs/heads/export-feature
Commit: de6d584809785576cf59cde5a83f05113fa6e92d
Parents: 78fd382
Author: George Reyes <gr...@apache.org>
Authored: Mon Oct 26 12:35:14 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Mon Oct 26 12:35:14 2015 -0700
----------------------------------------------------------------------
.../organizations/OrganizationResource.java | 7 +-
.../management/export/ExportFilter.java | 2 +-
.../management/export/ExportService.java | 2 +-
.../management/export/ExportServiceImpl.java | 280 +++++++++++--------
.../usergrid/management/export/export_v2.md | 5 +
5 files changed, 180 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
index a8cdfa8..789cb56 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/OrganizationResource.java
@@ -51,7 +51,7 @@ import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.export.ExportFilter;
import org.apache.usergrid.management.export.ExportFilterImpl;
import org.apache.usergrid.management.export.ExportService;
-import org.apache.usergrid.mq.Query;
+import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentials;
import org.apache.usergrid.rest.AbstractContextResource;
@@ -369,10 +369,9 @@ public class OrganizationResource extends AbstractContextResource {
exportFilter.setConnections( connectionSet );
//this references core, there needs to be a better exposed way to do this
//as well as a way to verify queries.
- exportFilter.setQuery(Query.fromQL( query ));
-
-
+ exportFilter.setQuery( Query.fromQL( query ));
+ return exportFilter;
}
@GET
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
index aff5f26..86fdf0b 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportFilter.java
@@ -19,7 +19,7 @@ package org.apache.usergrid.management.export;
import java.util.Set;
-import org.apache.usergrid.mq.Query;
+import org.apache.usergrid.persistence.Query;
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
index db6b105..045fa7f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -37,7 +37,7 @@ public interface ExportService {
* Schedules the export to execute using Objects that represent validated
* credentials and filters.
*/
- UUID schedule (Map<String,Object> json, ExportFilter exportFilter );
+ UUID schedule (Map<String,Object> json, ExportFilter exportFilter ) throws Exception;
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 11b5ae7..4835b63 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -311,65 +311,12 @@ public class ExportServiceImpl implements ExportService {
return;
}
-
//extracts the filter information
ExportFilter exportFilter = parseFilterInformation(jobData);
-
//Start the beginning of the flow.
exportApplicationsFromOrg( (UUID)config.get( "organizationId" ),config,jobExecution,s3Export,exportFilter );
- //we no longer have a concept of an application id. Just the filters from here on in.
-// else if ( config.get( "applicationId" ) == null ) {
-// //exports All the applications from an organization
-// try {
-// logger.debug( "starting export of all application from the following org uuid: "+config.get( "organizationId" ).toString() );
-// exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution, s3Export );
-// }
-// catch ( Exception e ) {
-// export.setErrorMessage( e.getMessage() );
-// export.setState( Export.State.FAILED );
-// em.update( export );
-// return;
-// }
-// }
-// else if ( config.get( "collectionName" ) == null ) {
-// //exports an Application from a single organization
-// try {
-// logger.debug( "Starting export of application: "+ config.get( "applicationId" ).toString());
-// exportApplicationFromOrg( ( UUID ) config.get( "organizationId" ),
-// ( UUID ) config.get( "applicationId" ), config, jobExecution, s3Export );
-// }
-// catch ( Exception e ) {
-// export.setErrorMessage( e.getMessage() );
-// export.setState( Export.State.FAILED );
-// em.update( export );
-// return;
-// }
-// }
-// else {
-// try {
-// //exports a single collection from an app org combo
-// try {
-// logger.debug( "Starting export of the following application collection: "+ config.get( "collectionName" ));
-// exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ), config, jobExecution,
-// s3Export );
-// }
-// catch ( Exception e ) {
-// export.setErrorMessage( e.getMessage() );
-// export.setState( Export.State.FAILED );
-// em.update( export );
-// return;
-// }
-// }
-// catch ( Exception e ) {
-// //if for any reason the backing up fails, then update the entity with a failed state.
-// export.setErrorMessage( e.getMessage() );
-// export.setState( Export.State.FAILED );
-// em.update( export );
-// return;
-// }
-// }
logger.debug( "finished the export job." );
updateExportStatus( export,Export.State.FINISHED,null );
}
@@ -476,45 +423,6 @@ public class ExportServiceImpl implements ExportService {
}
}
-
- /**
- * Exports a specific applications from an organization
- */
- private void exportApplicationFromOrg( UUID organizationUUID, UUID applicationId, final Map<String, Object> config,
- final JobExecution jobExecution, S3Export s3Export ) throws Exception {
-
- //retrieves export entity
- Export export = getExportEntity( jobExecution );
-
- ApplicationInfo application = managementService.getApplicationInfo( applicationId );
- String appFileName = prepareOutputFileName( application.getName(), null );
-
- Map ephemeral = collectionExportAndQuery( applicationId, config, export, jobExecution );
-
- fileTransfer( export, appFileName, ephemeral, config, s3Export );
- }
-
-
- /**
- * Exports a specific collection from an org-app combo.
- */
- //might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
- private void exportCollectionFromOrgApp( UUID applicationUUID, final Map<String, Object> config,
- final JobExecution jobExecution, S3Export s3Export ) throws Exception {
-
- //retrieves export entity
- Export export = getExportEntity( jobExecution );
- ApplicationInfo application = managementService.getApplicationInfo( applicationUUID );
-
- String appFileName = prepareOutputFileName( application.getName(), ( String ) config.get( "collectionName" ) );
-
-
- Map ephemeral = collectionExportAndQuery( applicationUUID, config, export, jobExecution );
-
- fileTransfer( export, appFileName, ephemeral, config, s3Export );
- }
-
-
/**
* Regulates how long to wait until the next heartbeat.
*/
@@ -706,11 +614,10 @@ public class ExportServiceImpl implements ExportService {
}
- protected JsonGenerator getJsonGenerator( File ephermal ) throws IOException {
+ protected JsonGenerator getJsonGenerator( OutputStream entityOutputStream ) throws IOException {
//TODO:shouldn't the below be UTF-16?
- FileOutputStream fileOutputStream = new FileOutputStream( ephermal );
- OutputStream entityOutputStream = new BufferedOutputStream( fileOutputStream );
+ //most systems are little endian.
JsonGenerator jg = jsonFactory.createGenerator( entityOutputStream, JsonEncoding.UTF16_LE );
jg.setPrettyPrinter( new MinimalPrettyPrinter( "" ) );
jg.setCodec( new ObjectMapper() );
@@ -736,6 +643,8 @@ public class ExportServiceImpl implements ExportService {
return outputFileName;
}
+//TODO: GREY need to create a method that will be called with a filename string then generate a output stream.
+ //That output stream will then be fed into the json generator. and added to the entitiesToExport array.
/**
* handles the query and export of collections
@@ -750,20 +659,26 @@ public class ExportServiceImpl implements ExportService {
int entitiesExportedCount = 0;
- //Could easily be converted to take in input streams. Just a harder refactor.
+ //TODO:GREY Add config to change path where this file is written.
+
List<File> entitiesToExport = new ArrayList<>( );
- //TODO:Add config to change path where this file is written.
File entityFileToBeExported = new File( "tempEntityExportPart1");
- JsonGenerator jg = getJsonGenerator( entityFileToBeExported );
entityFileToBeExported.deleteOnExit();
+ FileOutputStream fileEntityOutputStream = new FileOutputStream( entityFileToBeExported );
+ OutputStream entityOutputStream = new BufferedOutputStream( fileEntityOutputStream );
entitiesToExport.add( entityFileToBeExported );
+ JsonGenerator jg = getJsonGenerator( entityOutputStream );
+
+ //While this is more wordy it allows great seperation.
List<File> connectionsToExport = new ArrayList<>( );
File connectionFileToBeExported = new File ("tempConnectionExportPart1");
connectionFileToBeExported.deleteOnExit();
- JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
+ FileOutputStream fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported );
+ OutputStream connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream );
connectionsToExport.add( connectionFileToBeExported );
+ JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionOutputStream );
if(exportFilter.getCollections() == null || exportFilter.getCollections().isEmpty()) {
@@ -777,12 +692,12 @@ public class ExportServiceImpl implements ExportService {
//Query entity manager for the entities in a collection
Query query = null;
- if ( config.get( "query" ) == null ) {
+ if(exportFilter.getQuery() == null ) {
query = new Query();
}
else {
try {
- query = Query.fromQL( ( String ) config.get( "query" ) );
+ query = exportFilter.getQuery();
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -810,10 +725,11 @@ public class ExportServiceImpl implements ExportService {
if ( entitiesExportedCount % 1000 == 0 ) {
//Time to split files
currentFilePartIndex++;
- entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex );
- jg = getJsonGenerator( entityFileToBeExported );
- entityFileToBeExported.deleteOnExit();
- entitiesToExport.add( entityFileToBeExported );
+ File entityFileToBeExported2 = new File( "tempEntityExportPart" + currentFilePartIndex );
+ //TODO: UNCOMMENT THIS OR THE JSON GENERATION WON'T WORK.
+ //jg = getJsonGenerator( entityFileToBeExported2 );
+ entityFileToBeExported2.deleteOnExit();
+ entitiesToExport.add( entityFileToBeExported2 );
//It is quite likely that there are files that do not contain any connections and thus there will not
@@ -821,7 +737,7 @@ public class ExportServiceImpl implements ExportService {
// about that at this point.
connectionFileToBeExported = new File( "tempConnectionExportPart" + currentFilePartIndex );
connectionFileToBeExported.deleteOnExit();
- connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
+ //connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
connectionsToExport.add( connectionFileToBeExported );
}
}
@@ -836,12 +752,12 @@ public class ExportServiceImpl implements ExportService {
for( String collectionName : collectionSet ) {
//Query entity manager for the entities in a collection
Query query = null;
- if ( config.get( "query" ) == null ) {
+ if(exportFilter.getQuery() == null ) {
query = new Query();
}
else {
try {
- query = Query.fromQL( ( String ) config.get( "query" ) );
+ query = exportFilter.getQuery();
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -870,10 +786,14 @@ public class ExportServiceImpl implements ExportService {
//Time to split files
currentFilePartIndex++;
entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex );
- jg = getJsonGenerator( entityFileToBeExported );
entityFileToBeExported.deleteOnExit();
+ fileEntityOutputStream = new FileOutputStream( entityFileToBeExported );
+ entityOutputStream = new BufferedOutputStream( fileEntityOutputStream );
entitiesToExport.add( entityFileToBeExported );
+ jg = getJsonGenerator( entityOutputStream );
+
+
//It is quite likely that there are files that do not contain any connections and thus there will not
@@ -881,7 +801,9 @@ public class ExportServiceImpl implements ExportService {
// about that at this point.
connectionFileToBeExported = new File( "tempConnectionExportPart" + currentFilePartIndex );
connectionFileToBeExported.deleteOnExit();
- connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
+ fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported );
+ connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream );
+ connectionJsonGeneration = getJsonGenerator( connectionOutputStream );
connectionsToExport.add( connectionFileToBeExported );
}
}
@@ -897,6 +819,144 @@ public class ExportServiceImpl implements ExportService {
return filePointers;
}
+ //TODO: GREY need to create a method that will be called with a filename string then generate a output stream.
+ //That output stream will then be fed into the json generator. and added to the entitiesToExport array.
+
+ /**
+ * handles the query and export of collections
+ */
+ //TODO:Needs further refactoring.
+ protected Map collectionExportAndQuery2( UUID applicationUUID, final Map<String, Object> config, Export export,
+ final JobExecution jobExecution,ExportFilter exportFilter ) throws Exception {
+
+ EntityManager em = emf.getEntityManager( applicationUUID );
+ long starting_time = System.currentTimeMillis();
+ //The counter needs to be constant across collections since application exports do across collection aggregation.
+ int entitiesExportedCount = 0;
+
+
+ //TODO:GREY Add config to change path where this file is written.
+
+ List<File> entitiesToExport = new ArrayList<>( );
+ File entityFileToBeExported = new File( "tempEntityExportPart1");
+ entityFileToBeExported.deleteOnExit();
+ FileOutputStream fileEntityOutputStream = new FileOutputStream( entityFileToBeExported );
+ OutputStream entityOutputStream = new BufferedOutputStream( fileEntityOutputStream );
+ entitiesToExport.add( entityFileToBeExported );
+
+ JsonGenerator jg = getJsonGenerator( entityOutputStream );
+
+ //While this is more wordy it allows great seperation.
+ List<File> connectionsToExport = new ArrayList<>( );
+ File connectionFileToBeExported = new File ("tempConnectionExportPart1");
+ connectionFileToBeExported.deleteOnExit();
+ FileOutputStream fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported );
+ OutputStream connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream );
+ connectionsToExport.add( connectionFileToBeExported );
+
+ JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionOutputStream );
+
+
+ if(exportFilter.getCollections() == null || exportFilter.getCollections().isEmpty()) {
+ Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+
+ exportCollection( export, jobExecution, exportFilter, em, starting_time, entitiesExportedCount,
+ entitiesToExport, jg, connectionsToExport, connectionJsonGeneration, metadata.keySet() );
+ }
+ //handles the case where there are specific collections that need to be exported.
+ else{
+ Set<String> collectionSet = exportFilter.getCollections();
+
+ //loop through only the collection names
+ jg = exportCollection( export, jobExecution, exportFilter, em, starting_time, entitiesExportedCount,
+ entitiesToExport, jg, connectionsToExport, connectionJsonGeneration, collectionSet );
+ }
+ jg.flush();
+ jg.close();
+
+ HashMap<String,List> filePointers = new HashMap<>( );
+ filePointers.put( "entities",entitiesToExport );
+ filePointers.put( "connections",connectionsToExport );
+
+ return filePointers;
+ }
+
+
+ private JsonGenerator exportCollection( final Export export, final JobExecution jobExecution,
+ final ExportFilter exportFilter, final EntityManager em, long starting_time,
+ int entitiesExportedCount, final List<File> entitiesToExport,
+ JsonGenerator jg, final List<File> connectionsToExport,
+ JsonGenerator connectionJsonGeneration, final Set<String> collectionSet )
+ throws Exception {
+ final File entityFileToBeExported;
+ final FileOutputStream fileEntityOutputStream;
+ final OutputStream entityOutputStream;
+ final File connectionFileToBeExported;
+ final FileOutputStream fileConnectionOutputStream;
+ final OutputStream connectionOutputStream;
+ for( String collectionName : collectionSet ) {
+ //Query entity manager for the entities in a collection
+ Query query = null;
+ if(exportFilter.getQuery() == null ) {
+ query = new Query();
+ }
+ else {
+ try {
+ query = exportFilter.getQuery();
+ }
+ catch ( Exception e ) {
+ export.setErrorMessage( e.getMessage() );
+ }
+ }
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Level.ALL_PROPERTIES );
+ query.setCollection( collectionName );
+
+ Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
+
+ PagingResultsIterator itr = new PagingResultsIterator( entities );
+ //counter that will inform when we should split into another file.
+ int currentFilePartIndex = 1;
+
+ for ( Object e : itr ) {
+ starting_time = checkTimeDelta( starting_time, jobExecution );
+ Entity entity = ( Entity ) e;
+ jg.writeObject( entity );
+ saveCollectionMembers( jg, em, collectionName, entity );
+ saveConnections( entity, em, connectionJsonGeneration,exportFilter );
+ jg.writeRaw( '\n' );
+ jg.flush();
+ entitiesExportedCount++;
+ if ( entitiesExportedCount % 1000 == 0 ) {
+ //Time to split files
+ currentFilePartIndex++;
+ entityFileToBeExported = new File( "tempEntityExportPart" + currentFilePartIndex );
+ entityFileToBeExported.deleteOnExit();
+ fileEntityOutputStream = new FileOutputStream( entityFileToBeExported );
+ entityOutputStream = new BufferedOutputStream( fileEntityOutputStream );
+ entitiesToExport.add( entityFileToBeExported );
+
+ jg = getJsonGenerator( entityOutputStream );
+
+
+ //It is quite likely that there are files that do not contain any connections and thus there will not
+
+
+ //be anything to write to these empty connection files. Not entirely sure what to do
+ // about that at this point.
+ connectionFileToBeExported = new File( "tempConnectionExportPart" + currentFilePartIndex );
+ connectionFileToBeExported.deleteOnExit();
+ fileConnectionOutputStream = new FileOutputStream( connectionFileToBeExported );
+ connectionOutputStream = new BufferedOutputStream( fileConnectionOutputStream );
+ connectionJsonGeneration = getJsonGenerator( connectionOutputStream );
+ connectionsToExport.add( connectionFileToBeExported );
+ }
+ }
+ }
+ return jg;
+ }
+
+
private S3Export s3ExportDeterminator(final JobData jobData){
Object s3PlaceHolder = jobData.getProperty( "s3Export" );
S3Export s3Export = null;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6d5848/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
index ca86175..387d88a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/export_v2.md
@@ -170,6 +170,11 @@ These two lines take an application and search for a specific collection within
If the query is empty/null (need to apply the fix where queries can be null, currently they are always initialized) or if the query is "select *" then we go and search through the graph database for each entity. If the query is more sophisticated then we query elasticsearch for all relevant entity information. (Maybe a useful feature would be to perform a reindexing operation on each collection we wish to export. ) This would work exactly the same way a generic query performed at the rest tier would access the data.
+###What flow does it go through?
+Well first it aggregates a bidirection map of uuid's and application names. Then checks to see if there is any filter being applied. If not then we will export all the applications. If there is a filter then we only check each application that is present in the filter.
+
+Next we go to collections and check if the filter has listed any collection names to be exported. If there aren't, then we export every single collection. If there is a filter then we export and search for all the entities in the collection specific collections that are listed.
+