You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/04/07 20:21:27 UTC
[2/8] git commit: Making a switch to a ephemeral file transfers.
Fixed mock export. Made sure files delete when done with the ephemeral file.
Making a switch to a ephemeral file transfers.
Fixed mock export.
Made sure files delete when done with the ephemeral file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8a05f611
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8a05f611
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8a05f611
Branch: refs/pull/92/merge
Commit: 8a05f611e372e7a7ad4360e1eb617996b9f9c567
Parents: e3b79e8
Author: grey <gr...@apigee.com>
Authored: Mon Apr 7 09:53:32 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Mon Apr 7 09:53:32 2014 -0700
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 95 ++++++++++----------
.../usergrid/management/export/S3Export.java | 4 +-
.../management/export/S3ExportImpl.java | 6 +-
.../management/cassandra/MockS3ExportImpl.java | 49 +++++-----
4 files changed, 76 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index ac0eb20..bbbe414 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -17,15 +17,14 @@
package org.apache.usergrid.management.export;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.codehaus.jackson.JsonEncoding;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.map.ObjectMapper;
@@ -79,6 +78,7 @@ public class ExportServiceImpl implements ExportService {
private JsonFactory jsonFactory = new JsonFactory();
+
@Override
public UUID schedule( final Map<String, Object> config ) throws Exception {
ApplicationInfo defaultExportApp = null;
@@ -91,11 +91,10 @@ public class ExportServiceImpl implements ExportService {
EntityManager em = null;
try {
em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
- Set<String> collections = em.getApplicationCollections();
- if(!collections.contains( "exports" )){
+ Set<String> collections = em.getApplicationCollections();
+ if ( !collections.contains( "exports" ) ) {
em.createApplicationCollection( "exports" );
}
-
}
catch ( Exception e ) {
logger.error( "application doesn't exist within the current context" );
@@ -159,6 +158,7 @@ public class ExportServiceImpl implements ExportService {
return export.getState().toString();
}
+
@Override
public String getErrorMessage( final UUID appId, final UUID uuid ) throws Exception {
@@ -189,7 +189,7 @@ public class ExportServiceImpl implements ExportService {
@Override
public void doExport( final JobExecution jobExecution ) throws Exception {
Map<String, Object> config = ( Map<String, Object> ) jobExecution.getJobData().getProperty( "exportInfo" );
- Object s3PlaceHolder = jobExecution.getJobData().getProperty( "s3Export" ) ;
+ Object s3PlaceHolder = jobExecution.getJobData().getProperty( "s3Export" );
S3Export s3Export = null;
if ( config == null ) {
@@ -206,12 +206,14 @@ public class ExportServiceImpl implements ExportService {
export.setState( Export.State.STARTED );
em.update( export );
try {
- if(s3PlaceHolder != null) {
+ if ( s3PlaceHolder != null ) {
s3Export = ( S3Export ) s3PlaceHolder;
}
- else
+ else {
s3Export = new S3ExportImpl();
- }catch (Exception e){
+ }
+ }
+ catch ( Exception e ) {
logger.error( "S3Export doesn't exist" );
export.setErrorMessage( e.getMessage() );
export.setState( Export.State.FAILED );
@@ -228,7 +230,7 @@ public class ExportServiceImpl implements ExportService {
else if ( config.get( "applicationId" ) == null ) {
//exports All the applications from an organization
try {
- exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution,s3Export );
+ exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution, s3Export );
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -254,8 +256,8 @@ public class ExportServiceImpl implements ExportService {
try {
//exports a single collection from an app org combo
try {
- exportCollectionFromOrgApp( ( UUID ) config.get( "organizationId" ),
- ( UUID ) config.get( "applicationId" ), config, jobExecution,s3Export );
+ exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ), config, jobExecution,
+ s3Export );
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -307,7 +309,8 @@ public class ExportServiceImpl implements ExportService {
this.managementService = managementService;
}
- public Export getExportEntity ( final JobExecution jobExecution) throws Exception{
+
+ public Export getExportEntity( final JobExecution jobExecution ) throws Exception {
UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
EntityManager exportManager = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
@@ -315,6 +318,7 @@ public class ExportServiceImpl implements ExportService {
return exportManager.get( exportId, Export.class );
}
+
/**
* Exports All Applications from an Organization
*/
@@ -323,7 +327,7 @@ public class ExportServiceImpl implements ExportService {
//TODO: move extranous code out of these methods and make each one more distinct.
//retrieves export entity
- Export export = getExportEntity(jobExecution);
+ Export export = getExportEntity( jobExecution );
String appFileName = null;
BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organizationUUID );
@@ -337,26 +341,28 @@ public class ExportServiceImpl implements ExportService {
appFileName = prepareOutputFileName( "application", application.getValue(), null );
- ByteArrayOutputStream baos = collectionExportAndQuery(application.getKey(),config,export,jobExecution);
+ File ephemeral = collectionExportAndQuery( application.getKey(), config, export, jobExecution );
- fileTransfer( export,appFileName,baos,config, s3Export );
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
}
- public void fileTransfer(Export export,String appFileName,ByteArrayOutputStream baos,Map<String,Object> config, S3Export s3Export) {
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+
+ public void fileTransfer( Export export, String appFileName, File ephemeral, Map<String, Object> config,
+ S3Export s3Export ) {
try {
- s3Export.copyToS3( is, config, appFileName );
+ s3Export.copyToS3( ephemeral, config, appFileName );
+ if(ephemeral.exists())
+ ephemeral.delete();
}
catch ( Exception e ) {
+ export.setErrorMessage( e.getMessage() );
export.setState( Export.State.FAILED );
return;
}
}
-
-
/**
* Exports a specific applications from an organization
*/
@@ -364,15 +370,14 @@ public class ExportServiceImpl implements ExportService {
final JobExecution jobExecution, S3Export s3Export ) throws Exception {
//retrieves export entity
- Export export = getExportEntity(jobExecution);
+ Export export = getExportEntity( jobExecution );
ApplicationInfo application = managementService.getApplicationInfo( applicationId );
String appFileName = prepareOutputFileName( "application", application.getName(), null );
- ByteArrayOutputStream baos = collectionExportAndQuery(applicationId,config,export,jobExecution);
-
- fileTransfer( export,appFileName,baos,config, s3Export );
+ File ephemeral = collectionExportAndQuery( applicationId, config, export, jobExecution );
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
@@ -380,25 +385,20 @@ public class ExportServiceImpl implements ExportService {
* Exports a specific collection from an org-app combo.
*/
//might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
- private void exportCollectionFromOrgApp( UUID organizationUUID, UUID applicationUUID,
- final Map<String, Object> config, final JobExecution jobExecution, S3Export s3Export )
- throws Exception {
+ private void exportCollectionFromOrgApp( UUID applicationUUID, final Map<String, Object> config,
+ final JobExecution jobExecution, S3Export s3Export ) throws Exception {
//retrieves export entity
- Export export = getExportEntity(jobExecution);
+ Export export = getExportEntity( jobExecution );
ApplicationInfo application = managementService.getApplicationInfo( applicationUUID );
String appFileName = prepareOutputFileName( "application", application.getName(),
( String ) config.get( "collectionName" ) );
- ByteArrayOutputStream baos = collectionExportAndQuery(applicationUUID,config,export,jobExecution);
-
- //sets up the Inputstream for copying the method to s3.
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
-
- fileTransfer( export,appFileName,baos,config, s3Export );
+ File ephemeral = collectionExportAndQuery( applicationUUID, config, export, jobExecution );
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
@@ -525,10 +525,10 @@ public class ExportServiceImpl implements ExportService {
}
- protected JsonGenerator getJsonGenerator( ByteArrayOutputStream out ) throws IOException {
+ protected JsonGenerator getJsonGenerator( File ephermal ) throws IOException {
//TODO:shouldn't the below be UTF-16?
- JsonGenerator jg = jsonFactory.createJsonGenerator( out );
+ JsonGenerator jg = jsonFactory.createJsonGenerator( ephermal, JsonEncoding.UTF8 );
jg.setPrettyPrinter( new DefaultPrettyPrinter() );
jg.setCodec( new ObjectMapper() );
return jg;
@@ -556,23 +556,21 @@ public class ExportServiceImpl implements ExportService {
return outputFileName;
}
+
/**
* handles the query and export of collections
- * @param applicationUUID
- * @param config
- * @param export
- * @param jobExecution
- * @throws Exception
*/
//TODO:Needs further refactoring.
- protected ByteArrayOutputStream collectionExportAndQuery(UUID applicationUUID,final Map<String,Object> config,Export export,final JobExecution jobExecution) throws Exception{
+ protected File collectionExportAndQuery( UUID applicationUUID, final Map<String, Object> config, Export export,
+ final JobExecution jobExecution ) throws Exception {
EntityManager em = emf.getEntityManager( applicationUUID );
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
long starting_time = System.currentTimeMillis();
+ File ephemeral = new File( "tempExport" + starting_time );
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- JsonGenerator jg = getJsonGenerator( baos );
+
+ JsonGenerator jg = getJsonGenerator( ephemeral );
jg.writeStartArray();
@@ -618,10 +616,7 @@ public class ExportServiceImpl implements ExportService {
}
jg.writeEndArray();
jg.close();
- baos.flush();
- baos.close();
- return baos;
+ return ephemeral;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
index 63b10d9..97c9ee8 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
@@ -17,7 +17,7 @@
package org.apache.usergrid.management.export;
-import java.io.InputStream;
+import java.io.File;
import java.util.Map;
@@ -26,7 +26,7 @@ import java.util.Map;
*
*/
public interface S3Export {
- void copyToS3( InputStream inputStream, Map<String,Object> exportInfo, String filename );
+ void copyToS3( File ephemeral,Map<String,Object> exportInfo, String filename );
String getFilename ();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 521f962..4399ad7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -17,7 +17,7 @@
package org.apache.usergrid.management.export;
-import java.io.InputStream;
+import java.io.File;
import java.util.Map;
import java.util.Properties;
@@ -47,7 +47,7 @@ public class S3ExportImpl implements S3Export {
String fn;
@Override
- public void copyToS3( final InputStream inputStream, final Map<String,Object> exportInfo, String filename ) {
+ public void copyToS3( File ephemeral ,final Map<String,Object> exportInfo, String filename ) {
fn = filename;
@@ -90,7 +90,7 @@ public class S3ExportImpl implements S3Export {
try {
AsyncBlobStore blobStore = context.getAsyncBlobStore();
BlobBuilder blobBuilder =
- blobStore.blobBuilder( fn ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
+ blobStore.blobBuilder( fn ).payload( ephemeral ).calculateMD5().contentType( "text/plain" );
Blob blob = blobBuilder.build();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index adabf42..56e80b0 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -18,15 +18,10 @@ package org.apache.usergrid.management.cassandra;
import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.commons.io.FileUtils;
import org.apache.usergrid.management.export.S3Export;
@@ -42,28 +37,38 @@ public class MockS3ExportImpl implements S3Export {
}
@Override
- public void copyToS3( final InputStream inputStream, final Map<String,Object> exportInfo, String filename ) {
- Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
- int read = 0;
- byte[] bytes = new byte[1024];
- OutputStream outputStream = null;
+ public void copyToS3( File ephemeral, final Map<String,Object> exportInfo, String filename ) {
+ File verfiedData = new File( this.filename );
try {
- outputStream = new FileOutputStream( new File( this.filename ) );
+ FileUtils.copyFile(ephemeral,verfiedData);
}
- catch ( FileNotFoundException e ) {
+ catch ( IOException e ) {
e.printStackTrace();
}
- try {
- while ( ( read = ( inputStream.read( bytes ) ) ) != -1 ) {
- outputStream.write( bytes, 0, read );
- }
- }
- catch ( IOException e ) {
- e.printStackTrace();
- }
+ // Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
+// int read = 0;
+// byte[] bytes = new byte[1024];
+// OutputStream outputStream = null;
+//
+// try {
+// outputStream = new FileOutputStream( new File( this.filename ) );
+// }
+// catch ( FileNotFoundException e ) {
+// e.printStackTrace();
+// }
+//
+//
+// try {
+// while ( ( read = ( inputStream.read( bytes ) ) ) != -1 ) {
+// outputStream.write( bytes, 0, read );
+// }
+// }
+// catch ( IOException e ) {
+// e.printStackTrace();
+// }
}
@Override
@@ -71,6 +76,4 @@ public class MockS3ExportImpl implements S3Export {
return filename;
}
-// @Override
-// public void setFilename (String givenName) { filename = givenName; }
}