You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/04/07 20:21:26 UTC
[1/8] git commit: Ignore Netbeans project files.
Repository: incubator-usergrid
Updated Branches:
refs/pull/92/head e3b79e867 -> 537e8c5a1
refs/pull/92/merge 57cb46f11 -> 90e60ecb2 (forced update)
Ignore Netbeans project files.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/778511b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/778511b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/778511b7
Branch: refs/pull/92/merge
Commit: 778511b710df15b53a21db98b6a0f6aa0b393883
Parents: 1b4038a
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Apr 4 09:59:56 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Apr 4 09:59:56 2014 -0400
----------------------------------------------------------------------
.gitignore | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/778511b7/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 0da9432..1d8935a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,4 @@ stack/corepersistence/queryindex/nbactions.xml
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
+/portal/nbproject/private/
\ No newline at end of file
[6/8] git commit: Merge 9aeb66b0540e38e78a8db966bf3e8df9fe602f78 into
778511b710df15b53a21db98b6a0f6aa0b393883
Posted by sn...@apache.org.
Merge 9aeb66b0540e38e78a8db966bf3e8df9fe602f78 into 778511b710df15b53a21db98b6a0f6aa0b393883
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/90e60ecb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/90e60ecb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/90e60ecb
Branch: refs/pull/92/merge
Commit: 90e60ecb20d327a2fd42b7522e3243646da0ab40
Parents: 778511b 9aeb66b
Author: GERey <gr...@apigee.com>
Authored: Mon Apr 7 17:56:37 2014 +0000
Committer: GERey <gr...@apigee.com>
Committed: Mon Apr 7 17:56:37 2014 +0000
----------------------------------------------------------------------
.../usergrid/persistence/entities/Export.java | 70 --
.../organizations/OrganizationResource.java | 108 ++
.../applications/ApplicationResource.java | 27 -
.../rest/management/ExportResourceIT.java | 651 ++++++++++++
.../rest/management/ManagementResourceIT.java | 734 +++----------
.../usergrid/management/export/ExportJob.java | 1 +
.../management/export/ExportService.java | 5 +-
.../management/export/ExportServiceImpl.java | 347 +++---
.../usergrid/management/export/S3Export.java | 6 +-
.../management/export/S3ExportImpl.java | 9 +-
.../resources/usergrid-services-context.xml | 2 -
.../usergrid/ConcurrentServiceITSuite.java | 3 +-
.../management/cassandra/ExportServiceIT.java | 1004 ++++++++++++++++++
.../cassandra/ManagementServiceIT.java | 946 +----------------
.../management/cassandra/MockS3ExportImpl.java | 57 +-
.../security/tokens/TokenServiceIT.java | 3 +-
16 files changed, 2165 insertions(+), 1808 deletions(-)
----------------------------------------------------------------------
[8/8] git commit: Added delete on exit to ephemeral file.
Posted by sn...@apache.org.
Added delete on exit to ephemeral file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/537e8c5a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/537e8c5a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/537e8c5a
Branch: refs/pull/92/head
Commit: 537e8c5a1aa6282dcd75656ea387fe0066e4a4a9
Parents: 721d762
Author: grey <gr...@apigee.com>
Authored: Mon Apr 7 11:06:07 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Mon Apr 7 11:06:07 2014 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/management/export/ExportServiceImpl.java | 1 +
.../java/org/apache/usergrid/management/export/S3ExportImpl.java | 1 -
.../org/apache/usergrid/management/cassandra/MockS3ExportImpl.java | 1 -
3 files changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/537e8c5a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 61a416d..effacf0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -566,6 +566,7 @@ public class ExportServiceImpl implements ExportService {
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
long starting_time = System.currentTimeMillis();
File ephemeral = new File( "tempExport" + UUID.randomUUID() );
+ ephemeral.deleteOnExit();
JsonGenerator jg = getJsonGenerator( ephemeral );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/537e8c5a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 24130db..4399ad7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -99,7 +99,6 @@ public class S3ExportImpl implements S3Export {
logger.info( "Uploaded file etag=" + futureETag.get() );
- ephemeral.delete();
}
catch ( Exception e ) {
logger.error( "Error uploading to blob store", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/537e8c5a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index b3ec1cb..932d1f8 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -42,7 +42,6 @@ public class MockS3ExportImpl implements S3Export {
File verfiedData = new File( this.filename );
try {
FileUtils.copyFile(ephemeral,verfiedData);
- ephemeral.delete();
}
catch ( IOException e ) {
e.printStackTrace();
[2/8] git commit: Making a switch to a ephemeral file transfers.
Fixed mock export. Made sure files delete when done with the ephemeral file.
Posted by sn...@apache.org.
Making a switch to a ephemeral file transfers.
Fixed mock export.
Made sure files delete when done with the ephemeral file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8a05f611
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8a05f611
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8a05f611
Branch: refs/pull/92/merge
Commit: 8a05f611e372e7a7ad4360e1eb617996b9f9c567
Parents: e3b79e8
Author: grey <gr...@apigee.com>
Authored: Mon Apr 7 09:53:32 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Mon Apr 7 09:53:32 2014 -0700
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 95 ++++++++++----------
.../usergrid/management/export/S3Export.java | 4 +-
.../management/export/S3ExportImpl.java | 6 +-
.../management/cassandra/MockS3ExportImpl.java | 49 +++++-----
4 files changed, 76 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index ac0eb20..bbbe414 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -17,15 +17,14 @@
package org.apache.usergrid.management.export;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.codehaus.jackson.JsonEncoding;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.map.ObjectMapper;
@@ -79,6 +78,7 @@ public class ExportServiceImpl implements ExportService {
private JsonFactory jsonFactory = new JsonFactory();
+
@Override
public UUID schedule( final Map<String, Object> config ) throws Exception {
ApplicationInfo defaultExportApp = null;
@@ -91,11 +91,10 @@ public class ExportServiceImpl implements ExportService {
EntityManager em = null;
try {
em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
- Set<String> collections = em.getApplicationCollections();
- if(!collections.contains( "exports" )){
+ Set<String> collections = em.getApplicationCollections();
+ if ( !collections.contains( "exports" ) ) {
em.createApplicationCollection( "exports" );
}
-
}
catch ( Exception e ) {
logger.error( "application doesn't exist within the current context" );
@@ -159,6 +158,7 @@ public class ExportServiceImpl implements ExportService {
return export.getState().toString();
}
+
@Override
public String getErrorMessage( final UUID appId, final UUID uuid ) throws Exception {
@@ -189,7 +189,7 @@ public class ExportServiceImpl implements ExportService {
@Override
public void doExport( final JobExecution jobExecution ) throws Exception {
Map<String, Object> config = ( Map<String, Object> ) jobExecution.getJobData().getProperty( "exportInfo" );
- Object s3PlaceHolder = jobExecution.getJobData().getProperty( "s3Export" ) ;
+ Object s3PlaceHolder = jobExecution.getJobData().getProperty( "s3Export" );
S3Export s3Export = null;
if ( config == null ) {
@@ -206,12 +206,14 @@ public class ExportServiceImpl implements ExportService {
export.setState( Export.State.STARTED );
em.update( export );
try {
- if(s3PlaceHolder != null) {
+ if ( s3PlaceHolder != null ) {
s3Export = ( S3Export ) s3PlaceHolder;
}
- else
+ else {
s3Export = new S3ExportImpl();
- }catch (Exception e){
+ }
+ }
+ catch ( Exception e ) {
logger.error( "S3Export doesn't exist" );
export.setErrorMessage( e.getMessage() );
export.setState( Export.State.FAILED );
@@ -228,7 +230,7 @@ public class ExportServiceImpl implements ExportService {
else if ( config.get( "applicationId" ) == null ) {
//exports All the applications from an organization
try {
- exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution,s3Export );
+ exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution, s3Export );
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -254,8 +256,8 @@ public class ExportServiceImpl implements ExportService {
try {
//exports a single collection from an app org combo
try {
- exportCollectionFromOrgApp( ( UUID ) config.get( "organizationId" ),
- ( UUID ) config.get( "applicationId" ), config, jobExecution,s3Export );
+ exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ), config, jobExecution,
+ s3Export );
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -307,7 +309,8 @@ public class ExportServiceImpl implements ExportService {
this.managementService = managementService;
}
- public Export getExportEntity ( final JobExecution jobExecution) throws Exception{
+
+ public Export getExportEntity( final JobExecution jobExecution ) throws Exception {
UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
EntityManager exportManager = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
@@ -315,6 +318,7 @@ public class ExportServiceImpl implements ExportService {
return exportManager.get( exportId, Export.class );
}
+
/**
* Exports All Applications from an Organization
*/
@@ -323,7 +327,7 @@ public class ExportServiceImpl implements ExportService {
//TODO: move extranous code out of these methods and make each one more distinct.
//retrieves export entity
- Export export = getExportEntity(jobExecution);
+ Export export = getExportEntity( jobExecution );
String appFileName = null;
BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organizationUUID );
@@ -337,26 +341,28 @@ public class ExportServiceImpl implements ExportService {
appFileName = prepareOutputFileName( "application", application.getValue(), null );
- ByteArrayOutputStream baos = collectionExportAndQuery(application.getKey(),config,export,jobExecution);
+ File ephemeral = collectionExportAndQuery( application.getKey(), config, export, jobExecution );
- fileTransfer( export,appFileName,baos,config, s3Export );
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
}
- public void fileTransfer(Export export,String appFileName,ByteArrayOutputStream baos,Map<String,Object> config, S3Export s3Export) {
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+
+ public void fileTransfer( Export export, String appFileName, File ephemeral, Map<String, Object> config,
+ S3Export s3Export ) {
try {
- s3Export.copyToS3( is, config, appFileName );
+ s3Export.copyToS3( ephemeral, config, appFileName );
+ if(ephemeral.exists())
+ ephemeral.delete();
}
catch ( Exception e ) {
+ export.setErrorMessage( e.getMessage() );
export.setState( Export.State.FAILED );
return;
}
}
-
-
/**
* Exports a specific applications from an organization
*/
@@ -364,15 +370,14 @@ public class ExportServiceImpl implements ExportService {
final JobExecution jobExecution, S3Export s3Export ) throws Exception {
//retrieves export entity
- Export export = getExportEntity(jobExecution);
+ Export export = getExportEntity( jobExecution );
ApplicationInfo application = managementService.getApplicationInfo( applicationId );
String appFileName = prepareOutputFileName( "application", application.getName(), null );
- ByteArrayOutputStream baos = collectionExportAndQuery(applicationId,config,export,jobExecution);
-
- fileTransfer( export,appFileName,baos,config, s3Export );
+ File ephemeral = collectionExportAndQuery( applicationId, config, export, jobExecution );
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
@@ -380,25 +385,20 @@ public class ExportServiceImpl implements ExportService {
* Exports a specific collection from an org-app combo.
*/
//might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
- private void exportCollectionFromOrgApp( UUID organizationUUID, UUID applicationUUID,
- final Map<String, Object> config, final JobExecution jobExecution, S3Export s3Export )
- throws Exception {
+ private void exportCollectionFromOrgApp( UUID applicationUUID, final Map<String, Object> config,
+ final JobExecution jobExecution, S3Export s3Export ) throws Exception {
//retrieves export entity
- Export export = getExportEntity(jobExecution);
+ Export export = getExportEntity( jobExecution );
ApplicationInfo application = managementService.getApplicationInfo( applicationUUID );
String appFileName = prepareOutputFileName( "application", application.getName(),
( String ) config.get( "collectionName" ) );
- ByteArrayOutputStream baos = collectionExportAndQuery(applicationUUID,config,export,jobExecution);
-
- //sets up the Inputstream for copying the method to s3.
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
-
- fileTransfer( export,appFileName,baos,config, s3Export );
+ File ephemeral = collectionExportAndQuery( applicationUUID, config, export, jobExecution );
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
@@ -525,10 +525,10 @@ public class ExportServiceImpl implements ExportService {
}
- protected JsonGenerator getJsonGenerator( ByteArrayOutputStream out ) throws IOException {
+ protected JsonGenerator getJsonGenerator( File ephermal ) throws IOException {
//TODO:shouldn't the below be UTF-16?
- JsonGenerator jg = jsonFactory.createJsonGenerator( out );
+ JsonGenerator jg = jsonFactory.createJsonGenerator( ephermal, JsonEncoding.UTF8 );
jg.setPrettyPrinter( new DefaultPrettyPrinter() );
jg.setCodec( new ObjectMapper() );
return jg;
@@ -556,23 +556,21 @@ public class ExportServiceImpl implements ExportService {
return outputFileName;
}
+
/**
* handles the query and export of collections
- * @param applicationUUID
- * @param config
- * @param export
- * @param jobExecution
- * @throws Exception
*/
//TODO:Needs further refactoring.
- protected ByteArrayOutputStream collectionExportAndQuery(UUID applicationUUID,final Map<String,Object> config,Export export,final JobExecution jobExecution) throws Exception{
+ protected File collectionExportAndQuery( UUID applicationUUID, final Map<String, Object> config, Export export,
+ final JobExecution jobExecution ) throws Exception {
EntityManager em = emf.getEntityManager( applicationUUID );
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
long starting_time = System.currentTimeMillis();
+ File ephemeral = new File( "tempExport" + starting_time );
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- JsonGenerator jg = getJsonGenerator( baos );
+
+ JsonGenerator jg = getJsonGenerator( ephemeral );
jg.writeStartArray();
@@ -618,10 +616,7 @@ public class ExportServiceImpl implements ExportService {
}
jg.writeEndArray();
jg.close();
- baos.flush();
- baos.close();
- return baos;
+ return ephemeral;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
index 63b10d9..97c9ee8 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
@@ -17,7 +17,7 @@
package org.apache.usergrid.management.export;
-import java.io.InputStream;
+import java.io.File;
import java.util.Map;
@@ -26,7 +26,7 @@ import java.util.Map;
*
*/
public interface S3Export {
- void copyToS3( InputStream inputStream, Map<String,Object> exportInfo, String filename );
+ void copyToS3( File ephemeral,Map<String,Object> exportInfo, String filename );
String getFilename ();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 521f962..4399ad7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -17,7 +17,7 @@
package org.apache.usergrid.management.export;
-import java.io.InputStream;
+import java.io.File;
import java.util.Map;
import java.util.Properties;
@@ -47,7 +47,7 @@ public class S3ExportImpl implements S3Export {
String fn;
@Override
- public void copyToS3( final InputStream inputStream, final Map<String,Object> exportInfo, String filename ) {
+ public void copyToS3( File ephemeral ,final Map<String,Object> exportInfo, String filename ) {
fn = filename;
@@ -90,7 +90,7 @@ public class S3ExportImpl implements S3Export {
try {
AsyncBlobStore blobStore = context.getAsyncBlobStore();
BlobBuilder blobBuilder =
- blobStore.blobBuilder( fn ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
+ blobStore.blobBuilder( fn ).payload( ephemeral ).calculateMD5().contentType( "text/plain" );
Blob blob = blobBuilder.build();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index adabf42..56e80b0 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -18,15 +18,10 @@ package org.apache.usergrid.management.cassandra;
import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.commons.io.FileUtils;
import org.apache.usergrid.management.export.S3Export;
@@ -42,28 +37,38 @@ public class MockS3ExportImpl implements S3Export {
}
@Override
- public void copyToS3( final InputStream inputStream, final Map<String,Object> exportInfo, String filename ) {
- Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
- int read = 0;
- byte[] bytes = new byte[1024];
- OutputStream outputStream = null;
+ public void copyToS3( File ephemeral, final Map<String,Object> exportInfo, String filename ) {
+ File verfiedData = new File( this.filename );
try {
- outputStream = new FileOutputStream( new File( this.filename ) );
+ FileUtils.copyFile(ephemeral,verfiedData);
}
- catch ( FileNotFoundException e ) {
+ catch ( IOException e ) {
e.printStackTrace();
}
- try {
- while ( ( read = ( inputStream.read( bytes ) ) ) != -1 ) {
- outputStream.write( bytes, 0, read );
- }
- }
- catch ( IOException e ) {
- e.printStackTrace();
- }
+ // Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
+// int read = 0;
+// byte[] bytes = new byte[1024];
+// OutputStream outputStream = null;
+//
+// try {
+// outputStream = new FileOutputStream( new File( this.filename ) );
+// }
+// catch ( FileNotFoundException e ) {
+// e.printStackTrace();
+// }
+//
+//
+// try {
+// while ( ( read = ( inputStream.read( bytes ) ) ) != -1 ) {
+// outputStream.write( bytes, 0, read );
+// }
+// }
+// catch ( IOException e ) {
+// e.printStackTrace();
+// }
}
@Override
@@ -71,6 +76,4 @@ public class MockS3ExportImpl implements S3Export {
return filename;
}
-// @Override
-// public void setFilename (String givenName) { filename = givenName; }
}
[5/8] git commit: Added fix for ephemeral file having the same names.
Posted by sn...@apache.org.
Added fix for ephemeral file having the same names.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9aeb66b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9aeb66b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9aeb66b0
Branch: refs/pull/92/head
Commit: 9aeb66b0540e38e78a8db966bf3e8df9fe602f78
Parents: 8a05f61
Author: grey <gr...@apigee.com>
Authored: Mon Apr 7 10:46:27 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Mon Apr 7 10:46:27 2014 -0700
----------------------------------------------------------------------
.../apache/usergrid/management/export/ExportServiceImpl.java | 8 ++++----
.../org/apache/usergrid/management/export/S3ExportImpl.java | 1 +
.../usergrid/management/cassandra/MockS3ExportImpl.java | 1 +
3 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aeb66b0/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index bbbe414..61a416d 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -325,7 +325,6 @@ public class ExportServiceImpl implements ExportService {
private void exportApplicationsFromOrg( UUID organizationUUID, final Map<String, Object> config,
final JobExecution jobExecution, S3Export s3Export ) throws Exception {
- //TODO: move extranous code out of these methods and make each one more distinct.
//retrieves export entity
Export export = getExportEntity( jobExecution );
String appFileName = null;
@@ -352,8 +351,7 @@ public class ExportServiceImpl implements ExportService {
S3Export s3Export ) {
try {
s3Export.copyToS3( ephemeral, config, appFileName );
- if(ephemeral.exists())
- ephemeral.delete();
+
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -567,7 +565,7 @@ public class ExportServiceImpl implements ExportService {
EntityManager em = emf.getEntityManager( applicationUUID );
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
long starting_time = System.currentTimeMillis();
- File ephemeral = new File( "tempExport" + starting_time );
+ File ephemeral = new File( "tempExport" + UUID.randomUUID() );
JsonGenerator jg = getJsonGenerator( ephemeral );
@@ -611,10 +609,12 @@ public class ExportServiceImpl implements ExportService {
jg.writeObject( entity );
saveCollectionMembers( jg, em, ( String ) config.get( "collectionName" ), entity );
jg.writeEndObject();
+ jg.flush();
}
}
}
jg.writeEndArray();
+ jg.flush();
jg.close();
return ephemeral;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aeb66b0/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 4399ad7..24130db 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -99,6 +99,7 @@ public class S3ExportImpl implements S3Export {
logger.info( "Uploaded file etag=" + futureETag.get() );
+ ephemeral.delete();
}
catch ( Exception e ) {
logger.error( "Error uploading to blob store", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aeb66b0/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index 56e80b0..e765aea 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -42,6 +42,7 @@ public class MockS3ExportImpl implements S3Export {
File verfiedData = new File( this.filename );
try {
FileUtils.copyFile(ephemeral,verfiedData);
+ ephemeral.delete();
}
catch ( IOException e ) {
e.printStackTrace();
[7/8] git commit: Added ignore to some auto generated files in
intelliJ. Cleans up source tree commits. Removed commented out mocking code.
Posted by sn...@apache.org.
Added ignore to some auto generated files in intelliJ. Cleans up source tree commits.
Removed commented out mocking code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/721d7625
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/721d7625
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/721d7625
Branch: refs/pull/92/head
Commit: 721d7625a4dce54b9baae7c09f2f9f6d8c85cfd1
Parents: 9aeb66b
Author: grey <gr...@apigee.com>
Authored: Mon Apr 7 10:59:42 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Mon Apr 7 10:59:42 2014 -0700
----------------------------------------------------------------------
.gitignore | 2 ++
.../management/cassandra/MockS3ExportImpl.java | 23 --------------------
2 files changed, 2 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/721d7625/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 0da9432..f99c690 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,3 +35,5 @@ stack/corepersistence/queryindex/nbactions.xml
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
+sdks/android/usergrid-android-client.iml
+sdks/java/usergrid-java-client.iml
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/721d7625/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index e765aea..b3ec1cb 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -47,29 +47,6 @@ public class MockS3ExportImpl implements S3Export {
catch ( IOException e ) {
e.printStackTrace();
}
-
-
- // Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
-// int read = 0;
-// byte[] bytes = new byte[1024];
-// OutputStream outputStream = null;
-//
-// try {
-// outputStream = new FileOutputStream( new File( this.filename ) );
-// }
-// catch ( FileNotFoundException e ) {
-// e.printStackTrace();
-// }
-//
-//
-// try {
-// while ( ( read = ( inputStream.read( bytes ) ) ) != -1 ) {
-// outputStream.write( bytes, 0, read );
-// }
-// }
-// catch ( IOException e ) {
-// e.printStackTrace();
-// }
}
@Override
[3/8] git commit: Making a switch to a ephemeral file transfers.
Fixed mock export. Made sure files delete when done with the ephemeral file.
Posted by sn...@apache.org.
Making a switch to a ephemeral file transfers.
Fixed mock export.
Made sure files delete when done with the ephemeral file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8a05f611
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8a05f611
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8a05f611
Branch: refs/pull/92/head
Commit: 8a05f611e372e7a7ad4360e1eb617996b9f9c567
Parents: e3b79e8
Author: grey <gr...@apigee.com>
Authored: Mon Apr 7 09:53:32 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Mon Apr 7 09:53:32 2014 -0700
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 95 ++++++++++----------
.../usergrid/management/export/S3Export.java | 4 +-
.../management/export/S3ExportImpl.java | 6 +-
.../management/cassandra/MockS3ExportImpl.java | 49 +++++-----
4 files changed, 76 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index ac0eb20..bbbe414 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -17,15 +17,14 @@
package org.apache.usergrid.management.export;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.codehaus.jackson.JsonEncoding;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
import org.codehaus.jackson.map.ObjectMapper;
@@ -79,6 +78,7 @@ public class ExportServiceImpl implements ExportService {
private JsonFactory jsonFactory = new JsonFactory();
+
@Override
public UUID schedule( final Map<String, Object> config ) throws Exception {
ApplicationInfo defaultExportApp = null;
@@ -91,11 +91,10 @@ public class ExportServiceImpl implements ExportService {
EntityManager em = null;
try {
em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
- Set<String> collections = em.getApplicationCollections();
- if(!collections.contains( "exports" )){
+ Set<String> collections = em.getApplicationCollections();
+ if ( !collections.contains( "exports" ) ) {
em.createApplicationCollection( "exports" );
}
-
}
catch ( Exception e ) {
logger.error( "application doesn't exist within the current context" );
@@ -159,6 +158,7 @@ public class ExportServiceImpl implements ExportService {
return export.getState().toString();
}
+
@Override
public String getErrorMessage( final UUID appId, final UUID uuid ) throws Exception {
@@ -189,7 +189,7 @@ public class ExportServiceImpl implements ExportService {
@Override
public void doExport( final JobExecution jobExecution ) throws Exception {
Map<String, Object> config = ( Map<String, Object> ) jobExecution.getJobData().getProperty( "exportInfo" );
- Object s3PlaceHolder = jobExecution.getJobData().getProperty( "s3Export" ) ;
+ Object s3PlaceHolder = jobExecution.getJobData().getProperty( "s3Export" );
S3Export s3Export = null;
if ( config == null ) {
@@ -206,12 +206,14 @@ public class ExportServiceImpl implements ExportService {
export.setState( Export.State.STARTED );
em.update( export );
try {
- if(s3PlaceHolder != null) {
+ if ( s3PlaceHolder != null ) {
s3Export = ( S3Export ) s3PlaceHolder;
}
- else
+ else {
s3Export = new S3ExportImpl();
- }catch (Exception e){
+ }
+ }
+ catch ( Exception e ) {
logger.error( "S3Export doesn't exist" );
export.setErrorMessage( e.getMessage() );
export.setState( Export.State.FAILED );
@@ -228,7 +230,7 @@ public class ExportServiceImpl implements ExportService {
else if ( config.get( "applicationId" ) == null ) {
//exports All the applications from an organization
try {
- exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution,s3Export );
+ exportApplicationsFromOrg( ( UUID ) config.get( "organizationId" ), config, jobExecution, s3Export );
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -254,8 +256,8 @@ public class ExportServiceImpl implements ExportService {
try {
//exports a single collection from an app org combo
try {
- exportCollectionFromOrgApp( ( UUID ) config.get( "organizationId" ),
- ( UUID ) config.get( "applicationId" ), config, jobExecution,s3Export );
+ exportCollectionFromOrgApp( ( UUID ) config.get( "applicationId" ), config, jobExecution,
+ s3Export );
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -307,7 +309,8 @@ public class ExportServiceImpl implements ExportService {
this.managementService = managementService;
}
- public Export getExportEntity ( final JobExecution jobExecution) throws Exception{
+
+ public Export getExportEntity( final JobExecution jobExecution ) throws Exception {
UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
EntityManager exportManager = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
@@ -315,6 +318,7 @@ public class ExportServiceImpl implements ExportService {
return exportManager.get( exportId, Export.class );
}
+
/**
* Exports All Applications from an Organization
*/
@@ -323,7 +327,7 @@ public class ExportServiceImpl implements ExportService {
//TODO: move extranous code out of these methods and make each one more distinct.
//retrieves export entity
- Export export = getExportEntity(jobExecution);
+ Export export = getExportEntity( jobExecution );
String appFileName = null;
BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organizationUUID );
@@ -337,26 +341,28 @@ public class ExportServiceImpl implements ExportService {
appFileName = prepareOutputFileName( "application", application.getValue(), null );
- ByteArrayOutputStream baos = collectionExportAndQuery(application.getKey(),config,export,jobExecution);
+ File ephemeral = collectionExportAndQuery( application.getKey(), config, export, jobExecution );
- fileTransfer( export,appFileName,baos,config, s3Export );
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
}
- public void fileTransfer(Export export,String appFileName,ByteArrayOutputStream baos,Map<String,Object> config, S3Export s3Export) {
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+
+ public void fileTransfer( Export export, String appFileName, File ephemeral, Map<String, Object> config,
+ S3Export s3Export ) {
try {
- s3Export.copyToS3( is, config, appFileName );
+ s3Export.copyToS3( ephemeral, config, appFileName );
+ if(ephemeral.exists())
+ ephemeral.delete();
}
catch ( Exception e ) {
+ export.setErrorMessage( e.getMessage() );
export.setState( Export.State.FAILED );
return;
}
}
-
-
/**
* Exports a specific applications from an organization
*/
@@ -364,15 +370,14 @@ public class ExportServiceImpl implements ExportService {
final JobExecution jobExecution, S3Export s3Export ) throws Exception {
//retrieves export entity
- Export export = getExportEntity(jobExecution);
+ Export export = getExportEntity( jobExecution );
ApplicationInfo application = managementService.getApplicationInfo( applicationId );
String appFileName = prepareOutputFileName( "application", application.getName(), null );
- ByteArrayOutputStream baos = collectionExportAndQuery(applicationId,config,export,jobExecution);
-
- fileTransfer( export,appFileName,baos,config, s3Export );
+ File ephemeral = collectionExportAndQuery( applicationId, config, export, jobExecution );
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
@@ -380,25 +385,20 @@ public class ExportServiceImpl implements ExportService {
* Exports a specific collection from an org-app combo.
*/
//might be confusing, but uses the /s/ inclusion or exclusion nomenclature.
- private void exportCollectionFromOrgApp( UUID organizationUUID, UUID applicationUUID,
- final Map<String, Object> config, final JobExecution jobExecution, S3Export s3Export )
- throws Exception {
+ private void exportCollectionFromOrgApp( UUID applicationUUID, final Map<String, Object> config,
+ final JobExecution jobExecution, S3Export s3Export ) throws Exception {
//retrieves export entity
- Export export = getExportEntity(jobExecution);
+ Export export = getExportEntity( jobExecution );
ApplicationInfo application = managementService.getApplicationInfo( applicationUUID );
String appFileName = prepareOutputFileName( "application", application.getName(),
( String ) config.get( "collectionName" ) );
- ByteArrayOutputStream baos = collectionExportAndQuery(applicationUUID,config,export,jobExecution);
-
- //sets up the Inputstream for copying the method to s3.
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
-
- fileTransfer( export,appFileName,baos,config, s3Export );
+ File ephemeral = collectionExportAndQuery( applicationUUID, config, export, jobExecution );
+ fileTransfer( export, appFileName, ephemeral, config, s3Export );
}
@@ -525,10 +525,10 @@ public class ExportServiceImpl implements ExportService {
}
- protected JsonGenerator getJsonGenerator( ByteArrayOutputStream out ) throws IOException {
+ protected JsonGenerator getJsonGenerator( File ephermal ) throws IOException {
//TODO:shouldn't the below be UTF-16?
- JsonGenerator jg = jsonFactory.createJsonGenerator( out );
+ JsonGenerator jg = jsonFactory.createJsonGenerator( ephermal, JsonEncoding.UTF8 );
jg.setPrettyPrinter( new DefaultPrettyPrinter() );
jg.setCodec( new ObjectMapper() );
return jg;
@@ -556,23 +556,21 @@ public class ExportServiceImpl implements ExportService {
return outputFileName;
}
+
/**
* handles the query and export of collections
- * @param applicationUUID
- * @param config
- * @param export
- * @param jobExecution
- * @throws Exception
*/
//TODO:Needs further refactoring.
- protected ByteArrayOutputStream collectionExportAndQuery(UUID applicationUUID,final Map<String,Object> config,Export export,final JobExecution jobExecution) throws Exception{
+ protected File collectionExportAndQuery( UUID applicationUUID, final Map<String, Object> config, Export export,
+ final JobExecution jobExecution ) throws Exception {
EntityManager em = emf.getEntityManager( applicationUUID );
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
long starting_time = System.currentTimeMillis();
+ File ephemeral = new File( "tempExport" + starting_time );
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- JsonGenerator jg = getJsonGenerator( baos );
+
+ JsonGenerator jg = getJsonGenerator( ephemeral );
jg.writeStartArray();
@@ -618,10 +616,7 @@ public class ExportServiceImpl implements ExportService {
}
jg.writeEndArray();
jg.close();
- baos.flush();
- baos.close();
- return baos;
+ return ephemeral;
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
index 63b10d9..97c9ee8 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
@@ -17,7 +17,7 @@
package org.apache.usergrid.management.export;
-import java.io.InputStream;
+import java.io.File;
import java.util.Map;
@@ -26,7 +26,7 @@ import java.util.Map;
*
*/
public interface S3Export {
- void copyToS3( InputStream inputStream, Map<String,Object> exportInfo, String filename );
+ void copyToS3( File ephemeral,Map<String,Object> exportInfo, String filename );
String getFilename ();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 521f962..4399ad7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -17,7 +17,7 @@
package org.apache.usergrid.management.export;
-import java.io.InputStream;
+import java.io.File;
import java.util.Map;
import java.util.Properties;
@@ -47,7 +47,7 @@ public class S3ExportImpl implements S3Export {
String fn;
@Override
- public void copyToS3( final InputStream inputStream, final Map<String,Object> exportInfo, String filename ) {
+ public void copyToS3( File ephemeral ,final Map<String,Object> exportInfo, String filename ) {
fn = filename;
@@ -90,7 +90,7 @@ public class S3ExportImpl implements S3Export {
try {
AsyncBlobStore blobStore = context.getAsyncBlobStore();
BlobBuilder blobBuilder =
- blobStore.blobBuilder( fn ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
+ blobStore.blobBuilder( fn ).payload( ephemeral ).calculateMD5().contentType( "text/plain" );
Blob blob = blobBuilder.build();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a05f611/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index adabf42..56e80b0 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -18,15 +18,10 @@ package org.apache.usergrid.management.cassandra;
import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.commons.io.FileUtils;
import org.apache.usergrid.management.export.S3Export;
@@ -42,28 +37,38 @@ public class MockS3ExportImpl implements S3Export {
}
@Override
- public void copyToS3( final InputStream inputStream, final Map<String,Object> exportInfo, String filename ) {
- Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
- int read = 0;
- byte[] bytes = new byte[1024];
- OutputStream outputStream = null;
+ public void copyToS3( File ephemeral, final Map<String,Object> exportInfo, String filename ) {
+ File verfiedData = new File( this.filename );
try {
- outputStream = new FileOutputStream( new File( this.filename ) );
+ FileUtils.copyFile(ephemeral,verfiedData);
}
- catch ( FileNotFoundException e ) {
+ catch ( IOException e ) {
e.printStackTrace();
}
- try {
- while ( ( read = ( inputStream.read( bytes ) ) ) != -1 ) {
- outputStream.write( bytes, 0, read );
- }
- }
- catch ( IOException e ) {
- e.printStackTrace();
- }
+ // Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
+// int read = 0;
+// byte[] bytes = new byte[1024];
+// OutputStream outputStream = null;
+//
+// try {
+// outputStream = new FileOutputStream( new File( this.filename ) );
+// }
+// catch ( FileNotFoundException e ) {
+// e.printStackTrace();
+// }
+//
+//
+// try {
+// while ( ( read = ( inputStream.read( bytes ) ) ) != -1 ) {
+// outputStream.write( bytes, 0, read );
+// }
+// }
+// catch ( IOException e ) {
+// e.printStackTrace();
+// }
}
@Override
@@ -71,6 +76,4 @@ public class MockS3ExportImpl implements S3Export {
return filename;
}
-// @Override
-// public void setFilename (String givenName) { filename = givenName; }
}
[4/8] git commit: Added fix for ephemeral file having the same names.
Posted by sn...@apache.org.
Added fix for ephemeral file having the same names.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9aeb66b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9aeb66b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9aeb66b0
Branch: refs/pull/92/merge
Commit: 9aeb66b0540e38e78a8db966bf3e8df9fe602f78
Parents: 8a05f61
Author: grey <gr...@apigee.com>
Authored: Mon Apr 7 10:46:27 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Mon Apr 7 10:46:27 2014 -0700
----------------------------------------------------------------------
.../apache/usergrid/management/export/ExportServiceImpl.java | 8 ++++----
.../org/apache/usergrid/management/export/S3ExportImpl.java | 1 +
.../usergrid/management/cassandra/MockS3ExportImpl.java | 1 +
3 files changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aeb66b0/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index bbbe414..61a416d 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -325,7 +325,6 @@ public class ExportServiceImpl implements ExportService {
private void exportApplicationsFromOrg( UUID organizationUUID, final Map<String, Object> config,
final JobExecution jobExecution, S3Export s3Export ) throws Exception {
- //TODO: move extranous code out of these methods and make each one more distinct.
//retrieves export entity
Export export = getExportEntity( jobExecution );
String appFileName = null;
@@ -352,8 +351,7 @@ public class ExportServiceImpl implements ExportService {
S3Export s3Export ) {
try {
s3Export.copyToS3( ephemeral, config, appFileName );
- if(ephemeral.exists())
- ephemeral.delete();
+
}
catch ( Exception e ) {
export.setErrorMessage( e.getMessage() );
@@ -567,7 +565,7 @@ public class ExportServiceImpl implements ExportService {
EntityManager em = emf.getEntityManager( applicationUUID );
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
long starting_time = System.currentTimeMillis();
- File ephemeral = new File( "tempExport" + starting_time );
+ File ephemeral = new File( "tempExport" + UUID.randomUUID() );
JsonGenerator jg = getJsonGenerator( ephemeral );
@@ -611,10 +609,12 @@ public class ExportServiceImpl implements ExportService {
jg.writeObject( entity );
saveCollectionMembers( jg, em, ( String ) config.get( "collectionName" ), entity );
jg.writeEndObject();
+ jg.flush();
}
}
}
jg.writeEndArray();
+ jg.flush();
jg.close();
return ephemeral;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aeb66b0/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 4399ad7..24130db 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -99,6 +99,7 @@ public class S3ExportImpl implements S3Export {
logger.info( "Uploaded file etag=" + futureETag.get() );
+ ephemeral.delete();
}
catch ( Exception e ) {
logger.error( "Error uploading to blob store", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9aeb66b0/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index 56e80b0..e765aea 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -42,6 +42,7 @@ public class MockS3ExportImpl implements S3Export {
File verfiedData = new File( this.filename );
try {
FileUtils.copyFile(ephemeral,verfiedData);
+ ephemeral.delete();
}
catch ( IOException e ) {
e.printStackTrace();