You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/09/29 22:00:35 UTC
[09/15] usergrid git commit: Added fix where files over 1000 were
writing over the ones that were under 1000. Removed some commented out code
and added a test that was proving the issue.
Added fix where files over 1000 were writing over the ones that were under 1000.
Removed some commented out code and added a test that was proving the issue.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0c7203ab
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0c7203ab
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0c7203ab
Branch: refs/heads/export-feature
Commit: 0c7203abf37fb9f01faf4eb4416d0dc53da065a4
Parents: 1a5abf6
Author: GERey <gr...@apigee.com>
Authored: Tue Sep 22 16:06:31 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Tue Sep 22 16:06:31 2015 -0700
----------------------------------------------------------------------
.../management/export/AwsS3ExportImpl.java | 13 +-
.../management/export/ExportServiceImpl.java | 12 +-
.../management/export/MockExportServiceIT.java | 195 ++++++++++++++++++-
.../management/export/MockS3ExportImpl.java | 2 +
4 files changed, 200 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0c7203ab/stack/services/src/main/java/org/apache/usergrid/management/export/AwsS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/AwsS3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/AwsS3ExportImpl.java
index 067d1c4..2a8ea6e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/AwsS3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/AwsS3ExportImpl.java
@@ -129,13 +129,18 @@ public class AwsS3ExportImpl implements S3Export {
//for ( File fileToBeUploaded : ephemeral )
List<File> filePointersToEntityFiles= ( List<File> ) ephemeral.get( "entities" );
int index = 1;
- for(File entitiesToExport: filePointersToEntityFiles)
- write( entitiesToExport,"entities"+index+filename,bucketName );
+ for(File entitiesToExport: filePointersToEntityFiles) {
+ write( entitiesToExport, "entities" + index + filename, bucketName );
+ index++;
+ }
+ //index to label which parts of the while we are uploading and not uploading the same one repeatable.
index = 1;
List<File> filePointersToConnectionFiles= ( List<File> ) ephemeral.get( "connections" );
- for(File connectionsToExport: filePointersToConnectionFiles)
- write( connectionsToExport, "connections"+index+ filename, bucketName );
+ for(File connectionsToExport: filePointersToConnectionFiles) {
+ write( connectionsToExport, "connections" + index + filename, bucketName );
+ index++;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0c7203ab/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index f4cc242..da94d1c 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -623,6 +623,9 @@ public class ExportServiceImpl implements ExportService {
EntityManager em = emf.getEntityManager( applicationUUID );
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
long starting_time = System.currentTimeMillis();
+ //The counter needs to be constant across collections since application exports do across collection aggregation.
+ int entitiesExportedCount = 0;
+
//Could easily be converted to take in input streams. Just a harder refactor.
List<File> entitiesToExport = new ArrayList<>( );
@@ -637,11 +640,6 @@ public class ExportServiceImpl implements ExportService {
JsonGenerator connectionJsonGeneration = getJsonGenerator( connectionFileToBeExported );
connectionsToExport.add( connectionFileToBeExported );
- // JsonGenerator connectionJsonGeneration = getJsonGenerator( )//I believe this can take in a inputstream
-
- // jg.writeStartObject();
- // jg.writeObjectFieldStart( "collections" );
-
for ( String collectionName : metadata.keySet() ) {
if ( collectionName.equals( "exports" ) ) {
@@ -650,9 +648,6 @@ public class ExportServiceImpl implements ExportService {
//if the collection you are looping through doesn't match the name of the one you want. Don't export it.
if ( ( config.get( "collectionName" ) == null ) || collectionName.equalsIgnoreCase((String)config.get( "collectionName" ) ) ) {
- //write out the collection name at the start of the file
- //jg.writeArrayFieldStart( collectionName.toLowerCase() );
-
//Query entity manager for the entities in a collection
Query query = null;
if ( config.get( "query" ) == null ) {
@@ -674,7 +669,6 @@ public class ExportServiceImpl implements ExportService {
Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
PagingResultsIterator itr = new PagingResultsIterator( entities );
- int entitiesExportedCount = 0;
int currentFilePartIndex = 1;
for ( Object e : itr ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0c7203ab/stack/services/src/test/java/org/apache/usergrid/management/export/MockExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/MockExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/MockExportServiceIT.java
index 83f8a4c..68f453c 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/MockExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/MockExportServiceIT.java
@@ -387,20 +387,197 @@ public class MockExportServiceIT extends AbstractServiceIT {
@Test //Connections won't save when run with maven, but on local builds it will.
- public void testConnectionsOnApplicationEndpoint() throws Exception {
+ public void test2001EntitiesOnApplicationEndpoint() throws Exception {
String testFileName ="testConnectionsOnApplicationEndpoint.json";
-// File f = null;
+
+ S3Export s3Export = new MockS3ExportImpl( testFileName );
+
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder( appName );
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
+ // intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ int numberOfEntitiesToBeWritten = 999;
+ entity = new Entity[numberOfEntitiesToBeWritten];
+
+ // creates entities
+ for ( int i = 0; i < numberOfEntitiesToBeWritten; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test" + i + "@anuff.com" );
+ entity[i] = em.create( "users", userProperties );
+ }
+
+ setup.getEntityIndex().refresh( applicationId );
+
+ Thread.sleep( 1000 );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ //create and initialize jobData returned in JobExecution.
+ JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ exportService.doExport( jobExecution );
+
+ TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {};
+
+ int validEntitiesCounter = 0;
+ final InputStream in = new FileInputStream( "entities1"+testFileName );
+ try{
+ ObjectMapper mapper = new ObjectMapper();
+ JsonParser jp = new JsonFactory( ).createParser( in );
+
+
+ Iterator jsonIterator = mapper.readValues( jp, typeRef);
+ while(jsonIterator.hasNext()){
+ Object jsonEntity = jsonIterator.next();
+ assertNotNull( ( ( HashMap ) jsonEntity ).get( "uuid" ));
+ validEntitiesCounter++;
+ }
+ }finally{
+ in.close();
+// File exportedFile = new File("entities1"+testFileName);
+// exportedFile.delete();
+ }
+
+ //need to add three to all verifies because of the three default roles.
+ assertEquals( "There should have been "+(numberOfEntitiesToBeWritten+3)+" valid entities in the file",numberOfEntitiesToBeWritten+3,validEntitiesCounter );
+
+ //Read the connection entities and verify that they are correct
+
+// int validConnectionCounter = 0;
+// final InputStream connectionsInputStream = new FileInputStream( "connections1"+testFileName );
+// try{
+// ObjectMapper mapper = new ObjectMapper();
+// JsonParser jp = new JsonFactory( ).createParser( connectionsInputStream );
+//
+// Iterator jsonIterator = mapper.readValues( jp, typeRef);
+// while(jsonIterator.hasNext()){
+// Object jsonConnection = jsonIterator.next();
+// if(( ( HashMap ) jsonConnection ).get( entity[1].getUuid().toString() )!=null){
+// Object jsonEntityConnection = ( ( HashMap ) jsonConnection ).get( entity[1].getUuid().toString() );
+// assertNotNull( ( ( HashMap ) jsonEntityConnection ).get( "vibrations" ) );
+//
+// //String is stored in an array so we need to trim array fixings off the end of the string to compare.
+// String connectionUuid = ( ( HashMap ) jsonEntityConnection ).get( "vibrations" ).toString();
+// connectionUuid=connectionUuid.substring( 1,connectionUuid.length()-1 );
+// if(connectionUuid.equals( entity[0].getUuid().toString() ))
+// validConnectionCounter++;
+// }
+// else if(( ( HashMap ) jsonConnection ).get( entity[0].getUuid().toString() )!=null){
+// Object jsonEntityConnection = ( ( HashMap ) jsonConnection ).get( entity[0].getUuid().toString());
+// assertNotNull( ( ( HashMap ) jsonEntityConnection ).get( "vibrations" ) );
//
-// try {
-// f = new File( testFileName );
-// }
-// catch ( Exception e ) {
-// // consumed because this checks to see if the file exists.
-// // If it doesn't then don't do anything and carry on.
+// String connectionUuid = ( ( HashMap ) jsonEntityConnection ).get( "vibrations" ).toString();
+// connectionUuid=connectionUuid.substring( 1,connectionUuid.length()-1 );
+//
+// if(connectionUuid.equals( entity[1].getUuid().toString() ))
+// validConnectionCounter++;
+// }
+// }
+// }finally{
+// in.close();
+ File exportedFile = new File("connections1"+testFileName);
+ exportedFile.delete();
// }
-// f.deleteOnExit();
+//
+// assertEquals( "There should have been two valid connections in the file",2,validConnectionCounter );
+ }
+
+ @Test //Connections won't save when run with maven, but on local builds it will.
+ public void test1000EntitiesPerFile() throws Exception {
+
+ String testFileName ="testConnectionsOnApplicationEndpoint.json";
+
+ S3Export s3Export = new MockS3ExportImpl( testFileName );
+
+ ExportService exportService = setup.getExportService();
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder( appName );
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
+ // intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ int numberOfEntitiesToBeWritten = 1100;
+ entity = new Entity[numberOfEntitiesToBeWritten];
+
+ // creates entities
+ for ( int i = 0; i < numberOfEntitiesToBeWritten; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test" + i + "@anuff.com" );
+ entity[i] = em.create( "users", userProperties );
+ }
+
+ setup.getEntityIndex().refresh( applicationId );
+
+ Thread.sleep( 1000 );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ //create and initialize jobData returned in JobExecution.
+ JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ exportService.doExport( jobExecution );
+
+ TypeReference<HashMap<String, Object>> typeRef = new TypeReference<HashMap<String, Object>>() {};
+
+ int validEntitiesCounter = 0;
+ final InputStream in = new FileInputStream( "entities1"+testFileName );
+ try{
+ ObjectMapper mapper = new ObjectMapper();
+ JsonParser jp = new JsonFactory( ).createParser( in );
+
+
+ Iterator jsonIterator = mapper.readValues( jp, typeRef);
+ while(jsonIterator.hasNext()){
+ Object jsonEntity = jsonIterator.next();
+ assertNotNull( ( ( HashMap ) jsonEntity ).get( "uuid" ));
+ validEntitiesCounter++;
+ }
+ }finally{
+ in.close();
+ File exportedFile = new File("entities1"+testFileName);
+ exportedFile.delete();
+ exportedFile = new File("entities2"+testFileName);
+ exportedFile.delete();
+ }
+
+ //need to add three to all verifies because of the three default roles.
+ assertEquals( "There should have been 1000 valid entities in the file",1000,validEntitiesCounter );
+
+ //Delete the created connection files
+ File exportedFile = new File("connections1"+testFileName);
+ exportedFile.delete();
+ exportedFile = new File("connections2"+testFileName);
+ exportedFile.delete();
+ }
+
+ @Test //Connections won't save when run with maven, but on local builds it will.
+ public void testConnectionsOnApplicationEndpoint() throws Exception {
+
+ String testFileName ="testConnectionsOnApplicationEndpoint.json";
S3Export s3Export = new MockS3ExportImpl( testFileName );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0c7203ab/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
index 771027b..4d2e583 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/MockS3ExportImpl.java
@@ -53,6 +53,7 @@ public class MockS3ExportImpl implements S3Export {
File verifiedData = new File( "entities"+fileCounter+filename );
try {
FileUtils.copyFile( fileToBeWritten, verifiedData );
+ fileCounter++;
logger.info( "Copied file {} to {}", fileToBeWritten.getAbsolutePath(), verifiedData );
}
catch ( IOException e ) {
@@ -65,6 +66,7 @@ public class MockS3ExportImpl implements S3Export {
File verifiedData = new File("connections"+fileCounter+ filename );
try {
FileUtils.copyFile( fileToBeWritten, verifiedData );
+ fileCounter++;
logger.info( "Copied file {} to {}", fileToBeWritten.getAbsolutePath(), verifiedData );
}
catch ( IOException e ) {