You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/05/28 21:43:25 UTC
[49/50] [abbrv] incubator-usergrid git commit: Merge branch
'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
into two-dot-o-dev
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/265f84bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/265f84bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/265f84bf
Branch: refs/heads/two-dot-o-dev
Commit: 265f84bf6268b2a4cdb7a224be7b4d534b399df9
Parents: a0d9a61 ceadc6c
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu May 28 15:41:36 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu May 28 15:41:36 2015 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 9 +++++++--
stack/pom.xml | 2 +-
stack/rest/src/test/resources/log4j.properties | 3 ++-
stack/services/pom.xml | 10 ++++++++++
.../usergrid/management/export/S3ExportImpl.java | 7 +++++++
.../usergrid/services/assets/data/S3BinaryStore.java | 13 +++++++++++--
.../apache/usergrid/management/importer/S3Upload.java | 9 ++++++++-
.../usergrid/setup/ConcurrentProcessSingleton.java | 8 ++++++--
8 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/pom.xml
----------------------------------------------------------------------
diff --cc stack/pom.xml
index ecdefb1,ce78baf..349869c
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@@ -105,7 -105,7 +105,7 @@@
<hector-test-version>1.1-4</hector-test-version>
<jackson-version>1.9.9</jackson-version>
<jackson-2-version>2.3.3</jackson-2-version>
-- <jclouds.version>1.8.0</jclouds.version>
++ <jclouds.version>1.9.0</jclouds.version>
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/rest/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --cc stack/rest/src/test/resources/log4j.properties
index 27e35c8,27e35c8..e0aa09d
--- a/stack/rest/src/test/resources/log4j.properties
+++ b/stack/rest/src/test/resources/log4j.properties
@@@ -27,6 -27,6 +27,7 @@@ log4j.appender.stdout.layout=org.apache
log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) %c{1} - %m%n
log4j.logger.org.apache.usergrid=ERROR
++log4j.logger.org.apache.usergrid.setup=DEBUG
#log4j.logger.org.apache.usergrid.cassandra=DEBUG
#log4j.logger.org.apache.usergrid.persistence.cassandra=DEBUG
@@@ -66,7 -66,7 +67,7 @@@ log4j.logger.org.apache.usergrid.rest.f
#log4j.logger.org.apache.usergrid.services.notifiers=DEBUG
#log4j.logger.org.apache.usergrid.services.groups.users.devices=DEBUG
--log4j.logger.org.apache.usergrid.rest=INFO
++log4j.logger.org.apache.usergrid.rest=DEBUG
log4j.logger.org.apache.usergrid.rest.exceptions=DEBUG
#log4j.logger.org.apache.usergrid.rest.NotificationsIT=DEBUG
#log4j.logger.org.apache.usergrid.cassandra.CassandraResource=DEBUG
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/services/pom.xml
----------------------------------------------------------------------
diff --cc stack/services/pom.xml
index 820b99f,ace7e79..c6dd815
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@@ -248,6 -237,6 +248,16 @@@
<dependency>
<groupId>org.apache.jclouds</groupId>
<artifactId>jclouds-core</artifactId>
++ <exclusions>
++ <exclusion>
++ <artifactId>guice-assistedinject</artifactId>
++ <groupId>com.google.inject.extensions</groupId>
++ </exclusion>
++ <exclusion>
++ <artifactId>guice</artifactId>
++ <groupId>com.google.inject</groupId>
++ </exclusion>
++ </exclusions>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 4fff7c7,4fff7c7..f8005d1
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@@ -82,10 -82,10 +82,17 @@@ public class S3ExportImpl implements S3
try {
BlobStore blobStore = context.getBlobStore();
++
++ // need this for JClouds 1.7.x:
++// BlobBuilder blobBuilder =
++// blobStore.blobBuilder( filename ).payload( ephemeral ).calculateMD5().contentType( "application/json" );
++
++ // needed for JClouds 1.8.x:
BlobBuilder blobBuilder = blobStore.blobBuilder( filename )
.payload( ephemeral )
.contentMD5(Files.hash( ephemeral, Hashing.md5() ))
.contentType("application/json");
++
Blob blob = blobBuilder.build();
final String uploadedFile = blobStore.putBlob(
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
index a805e42,90bd24f..d028852
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
@@@ -17,19 -17,26 +17,18 @@@
package org.apache.usergrid.services.assets.data;
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import com.google.common.hash.HashCode;
-import com.google.common.hash.HashFunction;
+import com.google.common.collect.ImmutableSet;
- import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.google.common.io.Files;
+import com.google.inject.Module;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.utils.StringUtils;
import org.jclouds.ContextBuilder;
-import org.jclouds.blobstore.AsyncBlobStore;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
@@@ -41,16 -49,19 +40,15 @@@ import org.jclouds.netty.config.NettyPa
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.inject.Module;
+import java.io.*;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
-
public class S3BinaryStore implements BinaryStore {
private static final Iterable<? extends Module> MODULES = ImmutableSet
@@@ -109,19 -115,18 +107,25 @@@
long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB );
byte[] data = baos.toByteArray();
- final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
- fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
+ if ( written < FIVE_MB ) { // total smaller than 5mb
- String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
+ final String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
+ final String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
- if ( written < FIVE_MB ) { // total smaller than 5mb
+ final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
+ fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
BlobStore blobStore = getContext().getBlobStore();
++
++ // need this for JClouds 1.7.x:
++// BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
++// .payload( data ).calculateMD5().contentType( mimeType );
++
++ // need this for JClouds 1.8.x:
BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder(uploadFileName)
- .payload(data)
- .contentMD5(Hashing.md5().newHasher().putBytes( data ).hash())
- .contentType(mimeType);
+ .payload( data )
+ .contentMD5( Hashing.md5().newHasher().putBytes( data ).hash() )
+ .contentType( mimeType );
fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
@@@ -193,133 -224,5 +197,138 @@@
BlobStore blobStore = getContext().getBlobStore();
blobStore.removeBlob( bucketName, AssetUtils.buildAssetKey( appId, entity ) );
}
+
+ class UploadWorker implements Callable<Void> {
+
+ private UUID appId;
+ private Entity entity;
+ private InputStream inputStream;
+ private byte[] data;
+ private long written;
+
+
+ public UploadWorker( UUID appId, Entity entity, InputStream is, byte[] data, long written ) {
+ this.appId = appId;
+ this.entity = entity;
+ this.inputStream = is;
+ this.data = data;
+ this.written = written;
+ }
+
+ @Override
+ public Void call() {
+
+ LOG.debug( "Writing temp file for S3 upload" );
+
+ // determine max size file allowed, default to 50mb
+ long maxSizeBytes = 50 * FileUtils.ONE_MB;
+ String maxSizeMbString = properties.getProperty( "usergrid.binary.max-size-mb", "50" );
+ if (StringUtils.isNumeric( maxSizeMbString )) {
+ maxSizeBytes = Long.parseLong( maxSizeMbString ) * FileUtils.ONE_MB;
+ }
+
+ // always allow files up to 5mb
+ if (maxSizeBytes < 5 * FileUtils.ONE_MB ) {
+ maxSizeBytes = 5 * FileUtils.ONE_MB;
+ }
+
+ // write temporary file, slightly larger than our size limit
+ OutputStream os = null;
+ File tempFile;
+ try {
+ tempFile = File.createTempFile( entity.getUuid().toString(), "tmp" );
+ tempFile.deleteOnExit();
+ os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath() ) );
+ os.write( data );
+ written += data.length;
+ written += IOUtils.copyLarge( inputStream, os, 0, maxSizeBytes + 1 );
+
+ LOG.debug("Write temp file {} length {}", tempFile.getName(), written);
+
+ } catch ( IOException e ) {
+ throw new RuntimeException( "Error creating temp file", e );
+
+ } finally {
+ if ( os != null ) {
+ try {
+ os.flush();
+ } catch (IOException e) {
+ LOG.error( "Error flushing data to temporary upload file", e );
+ }
+ IOUtils.closeQuietly( os );
+ }
+ }
+
+ // if tempFile is too large, delete it, add error to entity file metadata and abort
+
+ Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
+
+ if ( tempFile.length() > maxSizeBytes ) {
+ LOG.debug("File too large. Temp file size (bytes) = {}, " +
+ "Max file size (bytes) = {} ", tempFile.length(), maxSizeBytes);
+ try {
+ EntityManager em = emf.getEntityManager( appId );
+ fileMetadata.put( "error", "Asset size " + tempFile.length()
+ + " is larger than max size of " + maxSizeBytes );
+ em.update( entity );
+ tempFile.delete();
+
+ } catch ( Exception e ) {
+ LOG.error( "Error updating entity with error message", e);
+ }
+ return null;
+ }
+
+ String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
+ String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
+
+ try { // start the upload
+
+ LOG.debug( "S3 upload thread started" );
+
+ BlobStore blobStore = getContext().getBlobStore();
+
++ // need this for JClouds 1.7.x:
++// BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
++// .payload( tempFile ).calculateMD5().contentType( mimeType );
++
++ // need this for JClouds 1.8.x:
+ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
+ .payload( tempFile )
+ .contentMD5( Files.hash( tempFile, Hashing.md5() ) )
+ .contentType( mimeType );
+
+ if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
+ bb.contentDisposition( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ).toString() );
+ }
+ final Blob blob = bb.build();
+
+ String md5sum = Hex.encodeHexString( blob.getMetadata().getContentMetadata().getContentMD5() );
+ fileMetadata.put( AssetUtils.CHECKSUM, md5sum );
+
+ LOG.debug( "S3 upload starting" );
+
+ String eTag = blobStore.putBlob( bucketName, blob );
+
+ LOG.debug( "S3 upload complete eTag=" + eTag);
+
+ // update entity with eTag
+ EntityManager em = emf.getEntityManager( appId );
+ fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
+ fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
+ fileMetadata.put( AssetUtils.E_TAG, eTag );
+ em.update( entity );
+ }
+ catch ( Exception e ) {
+ LOG.error( "error uploading", e );
+ }
+
+ if ( tempFile != null && tempFile.exists() ) {
+ tempFile.delete();
+ }
+
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
----------------------------------------------------------------------
diff --cc stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
index c56ff66,c56ff66..e76c90a
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
@@@ -87,13 -87,13 +87,20 @@@ public class S3Upload
String filename = fileNameIterator.next();
File uploadFile = new File( filename );
--
++
try {
BlobStore blobStore = context.getBlobStore();
++
++ // need this for JClouds 1.7.x:
++// BlobBuilder.PayloadBlobBuilder blobBuilder = blobStore.blobBuilder( filename )
++// .payload( uploadFile ).calculateMD5().contentType( "application/json" );
++
++ // needed for JClouds 1.8.x:
BlobBuilder blobBuilder = blobStore.blobBuilder( filename )
.payload( uploadFile )
.contentMD5(Files.hash( uploadFile, Hashing.md5()))
.contentType( "application/json" );
++
Blob blob = blobBuilder.build();
final String uploadedFile = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/265f84bf/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
----------------------------------------------------------------------
diff --cc stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
index 7cb5073,7cb5073..3b08bc8
--- a/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
+++ b/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
@@@ -89,14 -89,14 +89,18 @@@ public class ConcurrentProcessSingleto
// maybe delete existing column families and indexes
if ( CLEAN_STORAGE ) {
logger.info("Destroying current database");
-- schemaManager.destroy();
++ try {
++ schemaManager.destroy();
++ } catch ( Exception e ) {
++ logger.error("Exception on destroying current database, continuing", e);
++ }
}
// create our schema
logger.info("Creating database");
schemaManager.create();
-- logger.info("Populating database");
++ logger.info( "Populating database" );
schemaManager.populateBaseData();
// signal to other processes we've migrated, and they can proceed