You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/07/10 15:37:13 UTC
[07/50] [abbrv] incubator-usergrid git commit: Move to JClouds 1.7.1
for AWS S3 access to work around problems in OpenJDK 1.7 and Oracle JDK 8.
Move to JClouds 1.7.1 for AWS S3 access to work around problems in OpenJDK 1.7 and Oracle JDK 8.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e707628b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e707628b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e707628b
Branch: refs/heads/two-dot-o-import
Commit: e707628bf3a0aba3975717e47fab7951e58ae264
Parents: 5f4a66b
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri May 15 12:16:51 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri May 15 12:16:51 2015 -0400
----------------------------------------------------------------------
stack/pom.xml | 2 +-
.../rest/applications/ServiceResource.java | 2 +-
.../applications/assets/AssetResourceIT.java | 50 ++++++++++++----
.../services/assets/data/S3BinaryStore.java | 63 ++++++++++++--------
4 files changed, 79 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 4f50779..5c3d4fe 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -95,7 +95,7 @@
<hector-version>1.1-4</hector-version>
<hector-test-version>1.1-4</hector-test-version>
<jackson-version>1.9.9</jackson-version>
- <jclouds.version>1.6.2-incubating</jclouds.version>
+ <jclouds.version>1.7.1</jclouds.version>
<jersey-version>1.18</jersey-version>
<junit-version>4.11</junit-version>
<log4j-version>1.2.16</log4j-version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index d5ef311..3eee96b 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -595,7 +595,7 @@ public class ServiceResource extends AbstractContextResource {
ServiceResults serviceResults = executeServiceRequest( ui, response, ServiceAction.GET, null );
Entity entity = serviceResults.getEntity();
- LOG.info( "In AssetsResource.findAsset with id: {}, range: {}, modifiedSince: {}",
+ LOG.info( "In ServiceResource.executeStreamGet with id: {}, range: {}, modifiedSince: {}",
new Object[] { entityId, rangeHeader, modifiedSince } );
Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
index b41659d..666f95e 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
@@ -139,10 +139,14 @@ public class AssetResourceIT extends AbstractRestIT {
UserRepo.INSTANCE.load( resource(), access_token );
byte[] data = IOUtils.toByteArray( this.getClass().getResourceAsStream( "/file-bigger-than-5M" ) );
- FormDataMultiPart form = new FormDataMultiPart().field( "file", data, MediaType.MULTIPART_FORM_DATA_TYPE );
- JsonNode node = resource().path( "/test-organization/test-app/foos" ).queryParam( "access_token", access_token )
- .accept( MediaType.APPLICATION_JSON ).type( MediaType.MULTIPART_FORM_DATA )
+ FormDataMultiPart form = new FormDataMultiPart()
+ .field( "file", data, MediaType.MULTIPART_FORM_DATA_TYPE );
+
+ JsonNode node = resource().path( "/test-organization/test-app/foos" )
+ .queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.MULTIPART_FORM_DATA )
.post( JsonNode.class, form );
JsonNode idNode = node.get( "entities" ).get( 0 ).get( "uuid" );
@@ -151,25 +155,47 @@ public class AssetResourceIT extends AbstractRestIT {
logNode( node );
// get entity
- node = resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam( "access_token", access_token )
- .accept( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+ node = resource().path( "/test-organization/test-app/foos/" + uuid )
+ .queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_JSON_TYPE )
+ .get( JsonNode.class );
+
logNode( node );
assertEquals( "application/octet-stream", node.findValue( AssetUtils.CONTENT_TYPE ).getTextValue() );
assertEquals( 5324800, node.findValue( AssetUtils.CONTENT_LENGTH ).getIntValue() );
idNode = node.get( "entities" ).get( 0 ).get( "uuid" );
assertEquals( uuid, idNode.getTextValue() );
- // get data
- InputStream is =
- resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam( "access_token", access_token )
- .accept( MediaType.APPLICATION_OCTET_STREAM_TYPE ).get( InputStream.class );
+ int retries = 0;
+ boolean done = false;
+ byte[] foundData = new byte[0];
+
+ // retry until upload complete
+ while ( !done && retries < 30 ) {
+
+ // get data
+ try {
+ InputStream is = resource().path( "/test-organization/test-app/foos/" + uuid )
+ .queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_OCTET_STREAM_TYPE )
+ .get( InputStream.class );
+
+ foundData = IOUtils.toByteArray( is );
+ done = true;
+
+ } catch ( Exception intentiallyIgnored ) {}
+
+ Thread.sleep(1000);
+ retries++;
+ }
- byte[] foundData = IOUtils.toByteArray( is );
assertEquals( 5324800, foundData.length );
// delete
- node = resource().path( "/test-organization/test-app/foos/" + uuid ).queryParam( "access_token", access_token )
- .accept( MediaType.APPLICATION_JSON_TYPE ).delete( JsonNode.class );
+ node = resource().path( "/test-organization/test-app/foos/" + uuid )
+ .queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_JSON_TYPE )
+ .delete( JsonNode.class );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e707628b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
index e1748d3..29b5e47 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
@@ -104,7 +104,7 @@ public class S3BinaryStore implements BinaryStore {
@Override
public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws IOException {
- String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
+ final String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
ByteArrayOutputStream baos = new ByteArrayOutputStream();
long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB );
byte[] data = baos.toByteArray();
@@ -112,13 +112,13 @@ public class S3BinaryStore implements BinaryStore {
final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
- String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
+ final String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
if ( written < FIVE_MB ) { // total smaller than 5mb
BlobStore blobStore = getContext().getBlobStore();
- BlobBuilder.PayloadBlobBuilder bb =
- blobStore.blobBuilder( uploadFileName ).payload( data ).calculateMD5().contentType( mimeType );
+ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
+ .payload( data ).calculateMD5().contentType( mimeType );
fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
@@ -134,10 +134,11 @@ public class S3BinaryStore implements BinaryStore {
}
else { // bigger than 5mb... dump 5 mb tmp files and upload from them
- // todo: yes, AsyncBlobStore is deprecated, but there appears to be no replacement yet
- final AsyncBlobStore blobStore = getContext().getAsyncBlobStore();
+ // create temp file and copy entire file to that temp file
- File tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" );
+ LOG.debug( "Writing temp file for S3 upload" );
+
+ final File tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" );
tempFile.deleteOnExit();
OutputStream os = null;
try {
@@ -149,38 +150,52 @@ public class S3BinaryStore implements BinaryStore {
IOUtils.closeQuietly( os );
}
- BlobBuilder.PayloadBlobBuilder bb =
- blobStore.blobBuilder( uploadFileName ).payload( tempFile ).calculateMD5().contentType( mimeType );
-
fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
- if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
- bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString() );
- }
- final Blob blob = bb.build();
- final File finalTempFile = tempFile;
- final ListenableFuture<String> future =
- blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+ // JClouds no longer supports async blob store, so we have to do this fun stuff
- Runnable listener = new Runnable() {
+ LOG.debug( "Starting upload thread" );
+
+ Thread uploadThread = new Thread( new Runnable() {
@Override
public void run() {
try {
- String eTag = future.get();
+ LOG.debug( "S3 upload thread started" );
+
+ BlobStore blobStore = getContext().getBlobStore();
+
+ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
+ .payload( tempFile ).calculateMD5().contentType( mimeType );
+
+ if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
+ bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString() );
+ }
+ final Blob blob = bb.build();
+
+ String md5sum = Hex.encodeHexString( blob.getMetadata().getContentMetadata().getContentMD5() );
+ fileMetadata.put( AssetUtils.CHECKSUM, md5sum );
+
+ LOG.debug( "S3 upload starting" );
+
+ String eTag = blobStore.putBlob( bucketName, blob );
fileMetadata.put( AssetUtils.E_TAG, eTag );
+
+ LOG.debug( "S3 upload complete eTag=" + eTag);
+
EntityManager em = emf.getEntityManager( appId );
em.update( entity );
- finalTempFile.delete();
+ tempFile.delete();
}
catch ( Exception e ) {
LOG.error( "error uploading", e );
}
- if ( finalTempFile != null && finalTempFile.exists() ) {
- finalTempFile.delete();
+ if ( tempFile != null && tempFile.exists() ) {
+ tempFile.delete();
}
}
- };
- future.addListener( listener, executor );
+ });
+
+ uploadThread.start();
}
}