You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/10 21:25:59 UTC
[06/50] [abbrv] git commit: Correctly writes export file to s3 using
file stream buffers.
Correctly writes export file to s3 using file stream buffers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/15db8f4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/15db8f4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/15db8f4c
Branch: refs/pull/70/merge
Commit: 15db8f4cf5864ef0530071ed37883bac44f4a282
Parents: 68bd76b
Author: George Reyes <Ap...@George-Reyess-MacBook-Pro.local>
Authored: Wed Feb 12 17:09:14 2014 -0800
Committer: George Reyes <Ap...@George-Reyess-MacBook-Pro.local>
Committed: Wed Feb 12 17:09:14 2014 -0800
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 54 ++++++++++++--------
.../usergrid/management/export/S3Export.java | 4 +-
.../management/export/S3ExportImpl.java | 13 +++--
3 files changed, 45 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/15db8f4c/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
index c40f7f1..099f23e 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
@@ -1,9 +1,12 @@
package org.usergrid.management.export;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
-import java.io.PrintWriter;
+import java.io.InputStream;
+import java.io.ObjectOutputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -194,9 +197,14 @@ public class ExportServiceImpl implements ExportService{
return jobUUID;
}
-
+//TODO: make multipart streaming functional
+ //currently only stores the collection in memory then flushes it.
private void exportApplicationsForOrg( Map.Entry<UUID, String> organization,final ExportInfo config ) throws Exception {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos);
+
+
logger.info( "" + organization );
@@ -219,7 +227,7 @@ public class ExportServiceImpl implements ExportService{
String appFileName = prepareOutputFileName( "application", application.getValue() );
- JsonGenerator jg = getJsonGenerator( new File( appFileName ) );
+ JsonGenerator jg = getJsonGenerator( oos );
// load the dictionary
@@ -258,16 +266,17 @@ public class ExportServiceImpl implements ExportService{
nsEntity.setMetadata( "collections", collections );
jg.writeStartArray();
- jg.writeObject( nsEntity );
+ //jg.writeObject( nsEntity );
// Create a GENERATOR for the application collections.
//JsonGenerator collectionsJg = getJsonGenerator( createOutputFile( "collections", application.getValue() ) );
- String collectionsFilename = prepareOutputFileName( "collections","appDummyName" );
- JsonGenerator collectionsJg = getJsonGenerator( new File( collectionsFilename ) );
+ //String collectionsFilename = prepareOutputFileName( "collections","appDummyName" );
+ //JsonGenerator collectionsJg = getJsonGenerator( oos );
- collectionsJg.writeStartObject();
+ //collectionsJg.writeStartObject();
+ // jg.writeStartObject();
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
//don't need to echo as not a command line tool anymore
//echo( JsonUtils.mapToFormattedJsonString( metadata ) );
@@ -287,10 +296,10 @@ public class ExportServiceImpl implements ExportService{
for ( Entity entity : entities ) {
// Export the entity first and later the collections for
// this entity.
- jg.writeObject( entity );
+ //jg.writeObject( entity );
//echo( entity );
- saveCollectionMembers( collectionsJg, em, application.getValue(), entity );
+ saveCollectionMembers( jg, em, application.getValue(), entity );
}
//we're done
@@ -306,15 +315,20 @@ public class ExportServiceImpl implements ExportService{
}
// Close writer for the collections for this application.
- collectionsJg.writeEndObject();
- collectionsJg.close();
+ // collectionsJg.writeEndObject();
+ // collectionsJg.close();
// Close writer and file for this application.
+
+ // logger.warn();
jg.writeEndArray();
jg.close();
- s3Export.copyToS3( appFileName, config );
+ oos.flush();
+ oos.close();
+
+ InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+ s3Export.copyToS3( is, config );
//below line doesn't copy very good data anyways.
- //copyToS3( collectionsFilename, config );
}
}
@@ -335,7 +349,7 @@ public class ExportServiceImpl implements ExportService{
return;
}
- jg.writeFieldName( entity.getUuid().toString() );
+ // jg.writeFieldName( entity.getUuid().toString() );
jg.writeStartObject();
for ( String collectionName : collections ) {
@@ -368,15 +382,15 @@ public class ExportServiceImpl implements ExportService{
jg.writeEndObject();
}
- protected JsonGenerator getJsonGenerator( String outFile ) throws IOException {
- return getJsonGenerator( new File( outputDir, outFile ) );
- }
+ // protected JsonGenerator getJsonGenerator( String outFile ) throws IOException {
+ // return getJsonGenerator( new File( outputDir, outFile ) );
+ // }
- protected JsonGenerator getJsonGenerator( File outFile ) throws IOException {
+ protected JsonGenerator getJsonGenerator( ObjectOutputStream oos ) throws IOException {
//TODO:shouldn't the below be UTF-16?
- PrintWriter out = new PrintWriter( outFile, "UTF-8" );
- JsonGenerator jg = jsonFactory.createJsonGenerator( out );
+ //PrintWriter out = new PrintWriter( outFile, "UTF-8" );
+ JsonGenerator jg = jsonFactory.createJsonGenerator( oos );
jg.setPrettyPrinter( new DefaultPrettyPrinter() );
jg.setCodec( new ObjectMapper() );
return jg;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/15db8f4c/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
index 6895da5..cc4486c 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
@@ -1,6 +1,8 @@
package org.usergrid.management.export;
+import java.io.InputStream;
+
import org.usergrid.management.ExportInfo;
@@ -9,6 +11,6 @@ import org.usergrid.management.ExportInfo;
*
*/
public interface S3Export {
- void copyToS3( String fileName, ExportInfo exportInfo );
+ void copyToS3( InputStream inputStream, ExportInfo exportInfo );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/15db8f4c/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
index fc8e281..52afb71 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
@@ -1,7 +1,7 @@
package org.usergrid.management.export;
-import java.io.File;
+import java.io.InputStream;
import java.util.Properties;
import org.jclouds.ContextBuilder;
@@ -27,8 +27,9 @@ import com.google.inject.Module;
*
*/
public class S3ExportImpl implements S3Export {
+
@Override
- public void copyToS3( String fileName, final ExportInfo exportInfo ) {
+ public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo ) {
Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
/*won't need any of the properties as I have the export info*/
@@ -63,11 +64,12 @@ public class S3ExportImpl implements S3Export {
}
try {
- File file = new File( fileName );
+
+
AsyncBlobStore blobStore = context.getAsyncBlobStore();
BlobBuilder blobBuilder =
- blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType( "text/plain" )
- .contentLength( file.length() );
+ blobStore.blobBuilder( "test.json" ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
+
Blob blob = blobBuilder.build();
@@ -78,6 +80,7 @@ public class S3ExportImpl implements S3Export {
catch ( Exception e ) {
logger.error( "Error uploading to blob store", e );
}
+
}
}