You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/10 21:26:00 UTC
[07/50] [abbrv] git commit: Re included the code that would save
connections. Re included the code that would save dictionaries. Fix bug that
the output stream would attach junk data whenever writing data. Added
integration test ( with stripped credentia
Re included the code that would save connections.
Re included the code that would save dictionaries.
Fix bug that the output stream would attach junk data whenever writing data.
Added integration test ( with stripped credentials)
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2f678b31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2f678b31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2f678b31
Branch: refs/pull/70/merge
Commit: 2f678b3155a6e07fa17f37a9373d2a5e81d3d94b
Parents: 15db8f4
Author: George Reyes <Ap...@George-Reyess-MacBook-Pro.local>
Authored: Thu Feb 13 11:33:53 2014 -0800
Committer: George Reyes <Ap...@George-Reyess-MacBook-Pro.local>
Committed: Thu Feb 13 11:33:53 2014 -0800
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 97 ++++++++++++++----
.../cassandra/ManagementServiceIT.java | 101 ++++++++++++++++++-
2 files changed, 177 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f678b31/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
index 099f23e..0e70b29 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
@@ -6,7 +6,6 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.io.ObjectOutputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -15,8 +14,8 @@ import java.util.UUID;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.impl.DefaultPrettyPrinter;
import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.util.DefaultPrettyPrinter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -24,6 +23,7 @@ import org.usergrid.batch.service.SchedulerService;
import org.usergrid.management.ExportInfo;
import org.usergrid.management.ManagementService;
import org.usergrid.management.OrganizationInfo;
+import org.usergrid.persistence.ConnectionRef;
import org.usergrid.persistence.Entity;
import org.usergrid.persistence.EntityManager;
import org.usergrid.persistence.EntityManagerFactory;
@@ -196,13 +196,21 @@ public class ExportServiceImpl implements ExportService{
public UUID getJobUUID () {
return jobUUID;
}
-
+//write test checking to see what happens if the input stream is closed or wrong.
//TODO: make multipart streaming functional
//currently only stores the collection in memory then flushes it.
private void exportApplicationsForOrg( Map.Entry<UUID, String> organization,final ExportInfo config ) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- ObjectOutputStream oos = new ObjectOutputStream(baos);
+ //baos.reset();
+ //ObjectOutputStream oos = new ObjectOutputStream(baos);
+
+// OutputStreamWriter osw = new OutputStreamWriter(oos,"UTF-8");
+// PrintWriter out = new PrintWriter( osw );
+//
+// //oos.reset();
+//
+// Writer wrtJSon = new OutputStreamWriter( oos, "UTF-8" );
@@ -227,7 +235,7 @@ public class ExportServiceImpl implements ExportService{
String appFileName = prepareOutputFileName( "application", application.getValue() );
- JsonGenerator jg = getJsonGenerator( oos );
+ JsonGenerator jg = getJsonGenerator( baos );
// load the dictionary
@@ -266,7 +274,7 @@ public class ExportServiceImpl implements ExportService{
nsEntity.setMetadata( "collections", collections );
jg.writeStartArray();
- //jg.writeObject( nsEntity );
+ jg.writeObject( nsEntity );
// Create a GENERATOR for the application collections.
//JsonGenerator collectionsJg = getJsonGenerator( createOutputFile( "collections", application.getValue() ) );
@@ -314,19 +322,18 @@ public class ExportServiceImpl implements ExportService{
}
}
- // Close writer for the collections for this application.
- // collectionsJg.writeEndObject();
- // collectionsJg.close();
-
// Close writer and file for this application.
// logger.warn();
jg.writeEndArray();
jg.close();
- oos.flush();
- oos.close();
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+ baos.flush();
+ baos.close();
+
+
+ InputStream is = new ByteArrayInputStream( baos.toByteArray());
+ //InputStream is = new ObjectInputStream );
s3Export.copyToS3( is, config );
//below line doesn't copy very good data anyways.
}
@@ -373,10 +380,10 @@ public class ExportServiceImpl implements ExportService{
}
// Write connections
- //saveConnections( entity, em, jg );
+ saveConnections( entity, em, jg );
// Write dictionaries
- //saveDictionaries( entity, em, jg );
+ saveDictionaries( entity, em, jg );
// End the object if it was Started
jg.writeEndObject();
@@ -386,11 +393,67 @@ public class ExportServiceImpl implements ExportService{
// return getJsonGenerator( new File( outputDir, outFile ) );
// }
+ /** Persists the connection for this entity. */
+ private void saveDictionaries( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+
+ jg.writeFieldName( "dictionaries" );
+ jg.writeStartObject();
+
+ Set<String> dictionaries = em.getDictionaries( entity );
+ for ( String dictionary : dictionaries ) {
+
+ Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
+
+ // nothing to do
+ if ( dict.isEmpty() ) {
+ continue;
+ }
+
+ jg.writeFieldName( dictionary );
+
+ jg.writeStartObject();
+
+ for ( Map.Entry<Object, Object> entry : dict.entrySet() ) {
+ jg.writeFieldName( entry.getKey().toString() );
+ jg.writeObject( entry.getValue() );
+ }
+
+ jg.writeEndObject();
+ }
+ jg.writeEndObject();
+ }
+
+
+ /** Persists the connection for this entity. */
+ private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+
+ jg.writeFieldName( "connections" );
+ jg.writeStartObject();
+
+ Set<String> connectionTypes = em.getConnectionTypes( entity );
+ for ( String connectionType : connectionTypes ) {
+
+ jg.writeFieldName( connectionType );
+ jg.writeStartArray();
+
+ Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Results.Level.IDS );
+ List<ConnectionRef> connections = results.getConnections();
- protected JsonGenerator getJsonGenerator( ObjectOutputStream oos ) throws IOException {
+ for ( ConnectionRef connectionRef : connections ) {
+ jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
+ }
+
+ jg.writeEndArray();
+ }
+ jg.writeEndObject();
+ }
+
+
+ protected JsonGenerator getJsonGenerator(ByteArrayOutputStream out ) throws IOException {
//TODO:shouldn't the below be UTF-16?
//PrintWriter out = new PrintWriter( outFile, "UTF-8" );
- JsonGenerator jg = jsonFactory.createJsonGenerator( oos );
+
+ JsonGenerator jg = jsonFactory.createJsonGenerator( out );
jg.setPrettyPrinter( new DefaultPrettyPrinter() );
jg.setCodec( new ObjectMapper() );
return jg;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f678b31/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
index 80da981..2c630eb 100644
--- a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
@@ -26,6 +26,7 @@ import org.usergrid.management.OrganizationInfo;
import org.usergrid.management.UserInfo;
import org.usergrid.management.export.ExportJob;
import org.usergrid.management.export.S3Export;
+import org.usergrid.management.export.S3ExportImpl;
import org.usergrid.persistence.CredentialsInfo;
import org.usergrid.persistence.Entity;
import org.usergrid.persistence.EntityManager;
@@ -761,11 +762,16 @@ public class ManagementServiceIT {
S3Export s3Export = mock( S3Export.class );
- setup.getExportService().setS3Export( s3Export );
- setup.getExportService().doExport( exportInfo );
-
+ try {
+ setup.getExportService().setS3Export( s3Export );
+ setup.getExportService().doExport( exportInfo );
+ }catch (Exception e) {
+ assert(false);
+ }
+ assert(true);
}
+ //Tests to make sure we can call the job with mock data and it runs.
@Test
public void testExportDoJob() throws Exception {
@@ -806,7 +812,94 @@ public class ManagementServiceIT {
S3Export s3Export = mock( S3Export.class );
setup.getExportService().setS3Export( s3Export );
job.setExportService( setup.getExportService() );
- job.doJob( jobExecution );
+ try {
+ job.doJob( jobExecution );
+ }catch ( Exception e) {
+ assert( false );
+ }
+ assert(true);
+
+ }
+
+ //tests that with empty job data, the export still runs.
+ @Test
+ public void testExportEmptyJob() throws Exception {
+
+ JobData jobData = new JobData();
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ ExportJob job = new ExportJob();
+ S3Export s3Export = mock( S3Export.class );
+ setup.getExportService().setS3Export( s3Export );
+ job.setExportService( setup.getExportService() );
+ try {
+ job.doJob( jobExecution );
+ }catch ( Exception e) {
+ assert( false );
+ }
+ assert(true);
+
+ }
+
+ @Test
+ public void testNullJobExecution () {
+
+ JobData jobData = new JobData();
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ ExportJob job = new ExportJob();
+ S3Export s3Export = mock( S3Export.class );
+ setup.getExportService().setS3Export( s3Export );
+ job.setExportService( setup.getExportService() );
+ try {
+ job.doJob( jobExecution );
+ }catch ( Exception e) {
+ assert( false );
+ }
+ assert(true);
+
+ }
+
+ @Test
+ public void testS3IntegrationExport() throws Exception {
+
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ storage_info.put( "admin_token","insert_token_data_here" );
+ //TODO: always put dummy values here and ignore this test.
+ //TODO: add a ret for when s3 values are invalid.
+
+
+ properties.put( "storage_provider","s3");
+ properties.put( "storage_info",storage_info);
+
+ payload.put( "path", "test-organization/test-app/user");
+ payload.put( "properties", properties);
+
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+
+
+ S3Export s3Export = new S3ExportImpl(); //mock( S3Export.class );
+
+ //when(s3Export.copyToS3( any( InputStream.class),exportInfo ));
+
+ try {
+ setup.getExportService().setS3Export( s3Export );
+ setup.getExportService().doExport( exportInfo );
+ }catch (Exception e) {
+ e.printStackTrace();
+ assert(false);
+ }
+ assert(true);
}
+
}