You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2017/04/26 22:44:32 UTC
[2/2] usergrid git commit: Add a new Asset binary provider for Google
Cloud Storage. Refactor the BinaryStore implementation to follow a Factory
pattern.
Add a new Asset binary provider for Google Cloud Storage. Refactor the BinaryStore implementation to follow a Factory pattern.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bafd4627
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bafd4627
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bafd4627
Branch: refs/heads/master
Commit: bafd462744803f28175b13d0c205e3b029a6ac54
Parents: 949b465
Author: Michael Russo <ru...@google.com>
Authored: Wed Apr 26 15:43:40 2017 -0700
Committer: Michael Russo <ru...@google.com>
Committed: Wed Apr 26 15:43:40 2017 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 22 +-
.../test/resources/usergrid-test-context.xml | 6 +-
.../rest/applications/ServiceResource.java | 62 +--
.../applications/assets/AssetsResource.java | 38 +-
.../main/resources/usergrid-rest-context.xml | 15 +-
.../applications/assets/AssetResourceIT.java | 2 +-
.../applications/assets/AwsAssetResourceIT.java | 4 +-
.../assets/GoogleAssetResourceIT.java | 419 +++++++++++++++++++
.../applications/assets/aws/NoAWSCredsRule.java | 124 ------
.../assets/rules/NoAWSCredsRule.java | 136 ++++++
.../assets/rules/NoGoogleCredsRule.java | 118 ++++++
stack/services/pom.xml | 6 +
.../services/assets/BinaryStoreFactory.java | 83 ++++
.../services/assets/data/AWSBinaryStore.java | 320 ++++++++++++++
.../assets/data/AwsSdkS3BinaryStore.java | 317 --------------
.../services/assets/data/GoogleBinaryStore.java | 223 ++++++++++
.../assets/data/LocalFileBinaryStore.java | 17 +-
.../services/assets/data/S3BinaryStore.java | 346 ---------------
.../test/resources/usergrid-test-context.xml | 12 +-
19 files changed, 1381 insertions(+), 889 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 9c75ef0..0d1a193 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -565,13 +565,29 @@ usergrid.binary.bucketname=usergrid-binaries
#
usergrid.temp.files=/tmp/usergrid
-# Set the implementation of binary uploading to be used
-# Aws for aws s3 uploading, local
+# Set the implementation of binary uploading to be used. Valid values:
+#
+# local - uses the local file system and "usergrid.temp.files" specifies the location
+# aws - uses AWS Simple Storage Service (S3)
+# google - uses Google Cloud Platform Storage service (GCS)
+#
+#
+# If Google is specified, the following ENVIRONMENT variable must be set on the system running Usergrid:
+#
+# GOOGLE_APPLICATION_CREDENTIALS=/full/path/to/credentialfile.json
+#
+# See: https://developers.google.com/identity/protocols/application-default-credentials#howtheywork
+#
+#
+# If AWS is specified, the following JVM Arguments must be set:
+#
+# AWS_ACCESS_KEY_ID=awsKeyId
+# AWS_SECRET_KEY=awsKeySecret
+#
usergrid.binary.uploader=local
-
############################### Usergrid Admin ##############################
#
# Usergrid has a sysadmin user which has access to the complete Usergrid system
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/query-validator/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/query-validator/src/test/resources/usergrid-test-context.xml b/stack/query-validator/src/test/resources/usergrid-test-context.xml
index 7affe6d..19af73d 100644
--- a/stack/query-validator/src/test/resources/usergrid-test-context.xml
+++ b/stack/query-validator/src/test/resources/usergrid-test-context.xml
@@ -44,7 +44,11 @@
</bean>
- <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore"/>
+ <bean id="binaryStoreFactory" class="org.apache.usergrid.services.assets.BinaryStoreFactory">
+ <constructor-arg name="properties" ref="properties"/>
+ <constructor-arg name="entityManagerFactory" ref="entityManagerFactory"/>
+ <constructor-arg name="reposLocation" value="${usergrid.temp.files}"/>
+ </bean>
<bean id="setup" class="org.apache.usergrid.corepersistence.CpSetup">
<constructor-arg ref="entityManagerFactory"/>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index 3835b75..9373f5e 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.rest.applications;
import com.amazonaws.AmazonServiceException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+import com.google.cloud.storage.StorageException;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.management.OrganizationConfig;
import org.apache.usergrid.management.OrganizationConfigProps;
@@ -34,10 +35,8 @@ import org.apache.usergrid.rest.applications.assets.AssetsResource;
import org.apache.usergrid.rest.security.annotations.CheckPermissionsForPath;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.services.*;
-import org.apache.usergrid.services.assets.data.AssetUtils;
-import org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore;
-import org.apache.usergrid.services.assets.data.BinaryStore;
-import org.apache.usergrid.services.assets.data.LocalFileBinaryStore;
+import org.apache.usergrid.services.assets.BinaryStoreFactory;
+import org.apache.usergrid.services.assets.data.*;
import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
import org.apache.usergrid.utils.JsonUtils;
import org.glassfish.jersey.media.multipart.BodyPart;
@@ -46,6 +45,7 @@ import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeanInfoFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
@@ -70,15 +70,10 @@ public class ServiceResource extends AbstractContextResource {
protected static final Logger logger = LoggerFactory.getLogger( ServiceResource.class );
private static final String FILE_FIELD_NAME = "file";
-
- // @Autowired
private BinaryStore binaryStore;
@Autowired
- private LocalFileBinaryStore localFileBinaryStore;
-
- @Autowired
- private AwsSdkS3BinaryStore awsSdkS3BinaryStore;
+ private BinaryStoreFactory binaryStoreFactory;
protected ServiceManager services;
@@ -89,17 +84,6 @@ public class ServiceResource extends AbstractContextResource {
}
- public void setBinaryStore(String binaryStoreType){
-
- //TODO:GREY change this to be a property held elsewhere
- if(binaryStoreType.equals("local")){
- this.binaryStore = localFileBinaryStore;
- }
- else{
- this.binaryStore = awsSdkS3BinaryStore;
- }
- }
-
@Override
public void setParent( AbstractContextResource parent ) {
@@ -749,14 +733,8 @@ public class ServiceResource extends AbstractContextResource {
@Produces({MediaType.APPLICATION_JSON, "application/javascript"})
private ApiResponse executeMultiPart( UriInfo ui, String callback, FormDataMultiPart multiPart,
ServiceAction serviceAction ) throws Exception {
-
- //needed for testing
- if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
- this.binaryStore = localFileBinaryStore;
- }
- else{
- this.binaryStore = awsSdkS3BinaryStore;
- }
+ // needed for testing
+ this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
// collect form data values
List<BodyPart> bodyParts = multiPart.getBodyParts();
@@ -831,12 +809,8 @@ public class ServiceResource extends AbstractContextResource {
public Response uploadDataStream( @Context UriInfo ui, InputStream uploadedInputStream ) throws Exception {
//needed for testing
- if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
- this.binaryStore = localFileBinaryStore;
- }
- else{
- this.binaryStore = awsSdkS3BinaryStore;
- }
+ this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
+
ApiResponse response = createApiResponse();
response.setAction( "get" );
@@ -871,13 +845,8 @@ public class ServiceResource extends AbstractContextResource {
logger.trace( "ServiceResource.executeStreamGet" );
}
- //Needed for testing
- if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
- this.binaryStore = localFileBinaryStore;
- }
- else{
- this.binaryStore = awsSdkS3BinaryStore;
- }
+ // needed for testing
+ this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
ApiResponse response = createApiResponse();
response.setAction( "get" );
@@ -945,7 +914,6 @@ public class ServiceResource extends AbstractContextResource {
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
}
catch(AmazonServiceException ase){
-
if( ase.getStatusCode() > 499 ){
logger.error(ase.getMessage());
}else if(logger.isDebugEnabled()){
@@ -953,6 +921,14 @@ public class ServiceResource extends AbstractContextResource {
}
return Response.status(ase.getStatusCode()).build();
}
+ catch (StorageException se){
+ if( se.getCode() > 499 ){
+ logger.error(se.getMessage());
+ }else if(logger.isDebugEnabled()){
+ logger.debug(se.getMessage());
+ }
+ return Response.status(se.getCode()).build();
+ }
catch(RuntimeException re){
logger.error(re.getMessage());
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java
index b894205..f6fab41 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/assets/AssetsResource.java
@@ -26,10 +26,8 @@ import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.applications.ServiceResource;
import org.apache.usergrid.rest.security.annotations.CheckPermissionsForPath;
import org.apache.usergrid.rest.security.annotations.RequireApplicationAccess;
-import org.apache.usergrid.services.assets.data.AssetUtils;
-import org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore;
-import org.apache.usergrid.services.assets.data.BinaryStore;
-import org.apache.usergrid.services.assets.data.LocalFileBinaryStore;
+import org.apache.usergrid.services.assets.BinaryStoreFactory;
+import org.apache.usergrid.services.assets.data.*;
import org.apache.usergrid.utils.StringUtils;
import org.glassfish.jersey.media.multipart.FormDataParam;
import org.slf4j.Logger;
@@ -56,15 +54,10 @@ public class AssetsResource extends ServiceResource {
private static final Logger logger = LoggerFactory.getLogger( AssetsResource.class );
- //@Autowired
private BinaryStore binaryStore;
@Autowired
- private LocalFileBinaryStore localFileBinaryStore;
-
- @Autowired
- private AwsSdkS3BinaryStore awsSdkS3BinaryStore;
-
+ private BinaryStoreFactory binaryStoreFactory;
@Override
@@ -123,12 +116,8 @@ public class AssetsResource extends ServiceResource {
// @FormDataParam("file") FormDataContentDisposition fileDetail,
@PathParam("entityId") PathSegment entityId ) throws Exception {
- if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
- this.binaryStore = localFileBinaryStore;
- }
- else{
- this.binaryStore = awsSdkS3BinaryStore;
- }
+ // needed for testing
+ this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
if (uploadedInputStream != null ) {
UUID assetId = UUID.fromString( entityId.getPath() );
@@ -162,12 +151,8 @@ public class AssetsResource extends ServiceResource {
public Response uploadDataStream( @PathParam("entityId") PathSegment entityId, InputStream uploadedInputStream )
throws Exception {
- if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
- this.binaryStore = localFileBinaryStore;
- }
- else{
- this.binaryStore = awsSdkS3BinaryStore;
- }
+ // needed for testing
+ this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
UUID assetId = UUID.fromString( entityId.getPath() );
if (logger.isTraceEnabled()) {
@@ -191,12 +176,9 @@ public class AssetsResource extends ServiceResource {
public Response findAsset( @Context UriInfo ui, @QueryParam("callback") @DefaultValue("callback") String callback,
@PathParam("entityId") PathSegment entityId, @HeaderParam("range") String range,
@HeaderParam("if-modified-since") String modifiedSince ) throws Exception {
- if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
- this.binaryStore = localFileBinaryStore;
- }
- else{
- this.binaryStore = awsSdkS3BinaryStore;
- }
+
+ // needed for testing
+ this.binaryStore = binaryStoreFactory.getBinaryStore( properties.getProperty(PROPERTIES_USERGRID_BINARY_UPLOADER) );
UUID assetId = UUID.fromString( entityId.getPath() );
if (logger.isTraceEnabled()) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/main/resources/usergrid-rest-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/resources/usergrid-rest-context.xml b/stack/rest/src/main/resources/usergrid-rest-context.xml
index 5e925bf..2aaa93a 100644
--- a/stack/rest/src/main/resources/usergrid-rest-context.xml
+++ b/stack/rest/src/main/resources/usergrid-rest-context.xml
@@ -63,17 +63,10 @@
<property name="queueCapacity" value="25" />
</bean>
- <bean id="serviceResource"
- class="org.apache.usergrid.rest.applications.ServiceResource" scope="prototype">
- <property name="binaryStore" value="${usergrid.binary.uploader}"/>
+ <bean id="binaryStoreFactory" class="org.apache.usergrid.services.assets.BinaryStoreFactory">
+ <constructor-arg name="properties" ref="properties"/>
+ <constructor-arg name="entityManagerFactory" ref="entityManagerFactory"/>
+ <constructor-arg name="reposLocation" value="${usergrid.temp.files}"/>
</bean>
- <bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
- <property name="reposLocation" value="${usergrid.temp.files}"/>
- </bean>
-
- <bean id="awsSdkS3BinaryStore" class="org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore">
- </bean>
-
-
</beans>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
index 616d929..cc9c326 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AssetResourceIT.java
@@ -281,7 +281,7 @@ public class AssetResourceIT extends AbstractRestIT {
@Test
- public void largeFileInS3() throws Exception {
+ public void largeFile() throws Exception {
this.waitForQueueDrainAndRefreshIndex();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
index 4a9bfaa..196c662 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
@@ -21,7 +21,7 @@ import com.amazonaws.SDKGlobalConfiguration;
import net.jcip.annotations.NotThreadSafe;
import org.apache.commons.io.IOUtils;
-import org.apache.usergrid.rest.applications.assets.aws.NoAWSCredsRule;
+import org.apache.usergrid.rest.applications.assets.rules.NoAWSCredsRule;
import org.apache.usergrid.rest.test.resource.AbstractRestIT;
import org.apache.usergrid.rest.test.resource.model.ApiResponse;
import org.apache.usergrid.rest.test.resource.model.Entity;
@@ -58,7 +58,7 @@ public class AwsAssetResourceIT extends AbstractRestIT {
* Mark tests as ignored if no AWS creds are present
*/
@Rule
- public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
+ public NoAWSCredsRule credsRule = new NoAWSCredsRule();
@Before
public void setup(){
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java
new file mode 100644
index 0000000..811420e
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/GoogleAssetResourceIT.java
@@ -0,0 +1,419 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.applications.assets;
+
+
+import net.jcip.annotations.NotThreadSafe;
+import org.apache.commons.io.IOUtils;
+import org.apache.usergrid.rest.applications.assets.rules.NoAWSCredsRule;
+import org.apache.usergrid.rest.applications.assets.rules.NoGoogleCredsRule;
+import org.apache.usergrid.rest.test.resource.AbstractRestIT;
+import org.apache.usergrid.rest.test.resource.model.ApiResponse;
+import org.apache.usergrid.rest.test.resource.model.Entity;
+import org.apache.usergrid.services.assets.data.AssetUtils;
+import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
+import org.glassfish.jersey.media.multipart.FormDataMultiPart;
+import org.junit.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.ForbiddenException;
+import javax.ws.rs.InternalServerErrorException;
+import javax.ws.rs.NotFoundException;
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
+import static org.apache.usergrid.utils.MapUtils.hashMap;
+import static org.junit.Assert.*;
+
+
+@NotThreadSafe
+public class GoogleAssetResourceIT extends AbstractRestIT {
+
+ private Map<String, Object> originalProperties;
+ private static final Logger logger = LoggerFactory.getLogger(GoogleAssetResourceIT.class);
+
+ /**
+ * Mark tests as ignored if no credentials are present
+ */
+ @Rule
+ public NoGoogleCredsRule credsRule = new NoGoogleCredsRule();
+
+ @Before
+ public void setup() {
+ originalProperties = getRemoteTestProperties();
+ setTestProperty(PROPERTIES_USERGRID_BINARY_UPLOADER, "google");
+
+ }
+
+ @After
+ public void teardown() {
+ setTestProperties(originalProperties);
+ }
+
+
+ @Test
+ public void ensureMissingFileReturns404() {
+ Map<String, String> payload = hashMap("name", "assettest");
+ ApiResponse postResponse = pathResource(getOrgAppPath("missingFile")).post(payload);
+ UUID assetId = postResponse.getEntities().get(0).getUuid();
+ assertNotNull(assetId);
+
+ try {
+ pathResource(getOrgAppPath("missingFile/assettest")).getAssetAsStream(true);
+ fail("Should fail as there isn't an asset to retrieve.");
+ } catch (NotFoundException nfe) {
+ } catch (Exception e) {
+ logger.error("Unexpected exception", e);
+ fail("Shouldn't return any other kind of exception");
+ }
+
+ }
+
+ @Test
+ public void errorCheckingInvalidProperties() throws Exception {
+ Map<String, Object> errorTestProperties;
+ errorTestProperties = getRemoteTestProperties();
+ setTestProperty("usergrid.binary.bucketname", "xxx");
+
+ try {
+
+ Map<String, String> payload = hashMap("name", "assetname");
+ ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(payload);
+ UUID assetId = postResponse.getEntities().get(0).getUuid();
+ assertNotNull(assetId);
+
+ // post a binary asset to that entity
+
+ byte[] data = IOUtils.toByteArray(getClass().getResourceAsStream("/cassandra_eye.jpg"));
+ ApiResponse putResponse =
+ pathResource(getOrgAppPath("foos/" + assetId)).put(data, MediaType.APPLICATION_OCTET_STREAM_TYPE);
+
+
+ } catch (AwsPropertiesNotFoundException e) {
+ fail("Shouldn't interrupt runtime if access key isnt found.");
+ } catch (InternalServerErrorException uie) {
+ assertEquals(500, uie.getResponse().getStatus());
+ } finally {
+ setTestProperties(errorTestProperties);
+ }
+ }
+
+ @Test
+ public void errorCheckingInvalidPropertiesMultipartUpload() throws Exception {
+ Map<String, Object> errorTestProperties;
+ errorTestProperties = getRemoteTestProperties();
+ //test that we fail gracefully if we have missing properties
+ setTestProperty("usergrid.binary.bucketname", "xxx");
+
+ try {
+
+ byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/file-bigger-than-5M"));
+ FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+ ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(form);
+ UUID assetId = postResponse.getEntities().get(0).getUuid();
+ logger.info("Waiting for upload to finish...");
+ Thread.sleep(5000);
+
+ // check that entire file was uploaded
+
+ ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class);
+ logger.info("Upload complete!");
+ InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+ byte[] foundData = IOUtils.toByteArray(is);
+ assertEquals(data.length, foundData.length);
+
+ // delete file
+
+ pathResource(getOrgAppPath("foos/" + assetId)).delete();
+
+
+ } catch (AwsPropertiesNotFoundException e) {
+ fail("Shouldn't interrupt runtime if access key isnt found.");
+ } catch (ForbiddenException fe) {
+ assertEquals(403, fe.getResponse().getStatus());
+ } finally {
+ setTestProperties(errorTestProperties);
+ }
+ }
+
+ @Test
+ public void octetStreamOnDynamicEntity() throws Exception {
+
+ this.waitForQueueDrainAndRefreshIndex();
+
+ // post an asset entity
+
+ Map<String, String> payload = hashMap("name", "assetname");
+ ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(payload);
+ UUID assetId = postResponse.getEntities().get(0).getUuid();
+ assertNotNull(assetId);
+
+ // post a binary asset to that entity
+
+ byte[] data = IOUtils.toByteArray(getClass().getResourceAsStream("/cassandra_eye.jpg"));
+ ApiResponse putResponse = pathResource(getOrgAppPath("foos/" + assetId))
+ .put(data, MediaType.APPLICATION_OCTET_STREAM_TYPE);
+
+ // check that the asset entity has asset metadata
+
+ ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class);
+ Entity entity = getResponse.getEntities().get(0);
+ Map<String, Object> fileMetadata = (Map<String, Object>) entity.get("file-metadata");
+ Assert.assertEquals("image/jpeg", fileMetadata.get("content-type"));
+ Assert.assertEquals(7979, fileMetadata.get("content-length"));
+ assertEquals(assetId, entity.getUuid());
+
+ // get binary asset by UUID
+
+ InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+ byte[] foundData = IOUtils.toByteArray(is);
+ assertEquals(7979, foundData.length);
+
+ // get binary asset by name
+
+ is = pathResource(getOrgAppPath("foos/assetname")).getAssetAsStream();
+ foundData = IOUtils.toByteArray(is);
+ assertEquals(7979, foundData.length);
+ }
+
+
+ @Test
+ public void multipartPostFormOnDynamicEntity() throws Exception {
+
+ this.waitForQueueDrainAndRefreshIndex();
+
+ // post data larger than 5M
+
+ byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/file-bigger-than-5M"));
+ FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+ ApiResponse putResponse = pathResource(getOrgAppPath("foos")).post(form);
+ this.waitForQueueDrainAndRefreshIndex();
+
+ UUID assetId = putResponse.getEntities().get(0).getUuid();
+ assertNotNull(assetId);
+
+ // retry until upload complete and we can get the data
+
+ int retries = 0;
+ boolean done = false;
+ byte[] foundData = new byte[0];
+ while (!done && retries < 30) {
+
+ try {
+ InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+ foundData = IOUtils.toByteArray(is);
+ done = true;
+
+ } catch (Exception intentiallyIgnored) {
+ }
+
+ Thread.sleep(1000);
+ retries++;
+ }
+
+ // did we get expected number of bytes of data?
+
+ assertEquals(5324800, foundData.length);
+
+ pathResource(getOrgAppPath("foos/" + assetId)).delete();
+ }
+
+
+ @Test
+ public void multipartPutFormOnDynamicEntity() throws Exception {
+
+ this.waitForQueueDrainAndRefreshIndex();
+
+ // post an entity
+
+ Map<String, String> payload = hashMap("foo", "bar");
+ ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(payload);
+ UUID assetId = postResponse.getEntities().get(0).getUuid();
+ assertNotNull(assetId);
+
+ // post asset to that entity
+
+ byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/cassandra_eye.jpg"));
+ FormDataMultiPart form = new FormDataMultiPart()
+ .field("foo", "bar2")
+ .field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+ ApiResponse putResponse = pathResource(getOrgAppPath("foos/" + assetId)).put(form);
+ this.waitForQueueDrainAndRefreshIndex();
+
+ // get entity and check asset metadata
+
+ ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class);
+ Entity entity = getResponse.getEntities().get(0);
+ Map<String, Object> fileMetadata = (Map<String, Object>) entity.get("file-metadata");
+ long lastModified = Long.parseLong(fileMetadata.get(AssetUtils.LAST_MODIFIED).toString());
+
+ assertEquals(assetId, entity.getUuid());
+ assertEquals("bar2", entity.get("foo"));
+ assertEquals("image/jpeg", fileMetadata.get(AssetUtils.CONTENT_TYPE));
+ assertEquals(7979, fileMetadata.get(AssetUtils.CONTENT_LENGTH));
+
+ // get asset and check size
+
+ InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+ byte[] foundData = IOUtils.toByteArray(is);
+ assertEquals(7979, foundData.length);
+
+ // upload new asset to entity, then check that it was updated
+
+ ApiResponse putResponse2 = pathResource(getOrgAppPath("foos/" + assetId)).put(form);
+ entity = putResponse2.getEntities().get(0);
+ fileMetadata = (Map<String, Object>) entity.get("file-metadata");
+ long justModified = Long.parseLong(fileMetadata.get(AssetUtils.LAST_MODIFIED).toString());
+ assertNotEquals(lastModified, justModified);
+ }
+
+
+ @Test
+ public void largeFile() throws Exception {
+
+ this.waitForQueueDrainAndRefreshIndex();
+
+ // upload file larger than 5MB
+
+ byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/file-bigger-than-5M"));
+ FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+ ApiResponse postResponse = pathResource(getOrgAppPath("foos")).post(form);
+ UUID assetId = postResponse.getEntities().get(0).getUuid();
+ logger.info("Waiting for upload to finish...");
+ Thread.sleep(5000);
+
+ // check that entire file was uploaded
+
+ ApiResponse getResponse = pathResource(getOrgAppPath("foos/" + assetId)).get(ApiResponse.class);
+ logger.info("Upload complete!");
+ InputStream is = pathResource(getOrgAppPath("foos/" + assetId)).getAssetAsStream();
+ byte[] foundData = IOUtils.toByteArray(is);
+ assertEquals(data.length, foundData.length);
+
+ // delete file
+
+ pathResource(getOrgAppPath("foos/" + assetId)).delete();
+ }
+
+ @Test
+ public void fileTooLargeShouldResultInError() throws Exception {
+
+ this.waitForQueueDrainAndRefreshIndex();
+
+ // set max file size down to 5mb
+ setTestProperty("usergrid.binary.max-size-mb", "5");
+
+ try {
+
+ // upload a file larger than 6mb
+
+ byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/ship-larger-than-6mb.gif"));
+ FormDataMultiPart form = new FormDataMultiPart().field("file", data, MediaType.MULTIPART_FORM_DATA_TYPE);
+ ApiResponse postResponse = pathResource(getOrgAppPath("bars")).post(form);
+ UUID assetId = postResponse.getEntities().get(0).getUuid();
+
+ String errorMessage = null;
+ logger.info("Waiting for upload to finish...");
+ Thread.sleep(1000);
+
+ // attempt to get asset entity, it should contain error
+
+ waitForQueueDrainAndRefreshIndex();
+ ApiResponse getResponse = pathResource(getOrgAppPath("bars/" + assetId)).get(ApiResponse.class);
+ Map<String, Object> fileMetadata = (Map<String, Object>) getResponse.getEntities().get(0).get("file-metadata");
+ assertNotNull(fileMetadata);
+ assertNotNull(fileMetadata.get("error"));
+ assertTrue(fileMetadata.get("error").toString().startsWith("Asset size "));
+
+ } finally {
+
+ // set max upload size back to default 25mb
+ setTestProperties(originalProperties);
+ }
+ }
+
+ /**
+ * Deleting a connection to an asset should not delete the asset or the asset's data
+ */
+ @Test
+ public void deleteConnectionToAsset() throws IOException {
+
+ this.waitForQueueDrainAndRefreshIndex();
+
+ // create the entity that will be the asset, an image
+
+ Map<String, String> payload = hashMap("name", "cassandra_eye.jpg");
+ ApiResponse postReponse = pathResource(getOrgAppPath("foos")).post(payload);
+ final UUID uuid = postReponse.getEntities().get(0).getUuid();
+
+ // post image data to the asset entity
+
+ byte[] data = IOUtils.toByteArray(this.getClass().getResourceAsStream("/cassandra_eye.jpg"));
+ pathResource(getOrgAppPath("foos/" + uuid)).put(data, MediaType.APPLICATION_OCTET_STREAM_TYPE);
+
+ // create an imagegallery entity
+
+ Map<String, String> imageGalleryPayload = hashMap("name", "my image gallery");
+
+ ApiResponse postResponse2 = pathResource(getOrgAppPath("imagegalleries")).post(imageGalleryPayload);
+ UUID imageGalleryId = postResponse2.getEntities().get(0).getUuid();
+
+ // connect imagegallery to asset
+
+ ApiResponse connectResponse = pathResource(
+ getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/" + uuid)).post(ApiResponse.class);
+ this.waitForQueueDrainAndRefreshIndex();
+
+ // verify connection from imagegallery to asset
+
+ ApiResponse containsResponse = pathResource(
+ getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/")).get(ApiResponse.class);
+ assertEquals(uuid, containsResponse.getEntities().get(0).getUuid());
+
+ // delete the connection
+
+ pathResource(getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/" + uuid)).delete();
+ this.waitForQueueDrainAndRefreshIndex();
+
+ // verify that connection is gone
+
+ ApiResponse listResponse = pathResource(
+ getOrgAppPath("imagegalleries/" + imageGalleryId + "/contains/")).get(ApiResponse.class);
+ assertEquals(0, listResponse.getEntityCount());
+
+ // asset should still be there
+
+ ApiResponse getResponse2 = pathResource(getOrgAppPath("foos/" + uuid)).get(ApiResponse.class);
+ Entity entity = getResponse2.getEntities().get(0);
+ Map<String, Object> fileMetadata = (Map<String, Object>) entity.get("file-metadata");
+
+ Assert.assertEquals("image/jpeg", fileMetadata.get(AssetUtils.CONTENT_TYPE));
+ Assert.assertEquals(7979, fileMetadata.get(AssetUtils.CONTENT_LENGTH));
+ assertEquals(uuid, entity.getUuid());
+
+ // asset data should still be there
+
+ InputStream assetIs = pathResource(getOrgAppPath("foos/" + uuid)).getAssetAsStream();
+ byte[] foundData = IOUtils.toByteArray(assetIs);
+ assertEquals(7979, foundData.length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java
deleted file mode 100644
index 40a6b45..0000000
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/aws/NoAWSCredsRule.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.rest.applications.assets.aws;
-
-
-import com.amazonaws.AmazonClientException;
-import com.amazonaws.SDKGlobalConfiguration;
-import org.apache.usergrid.rest.test.resource.AbstractRestIT;
-import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
-import org.junit.Assume;
-import org.junit.internal.runners.model.MultipleFailureException;
-import org.junit.rules.TestRule;
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-import org.springframework.beans.factory.annotation.Autowired;
-
-import java.util.Map;
-import java.util.Properties;
-
-
-/**
- * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue
- * https://github.com/junit-team/junit/issues/116
- *
- * Until then, simply marks as passed, which is a bit dangerous
- */
-public class NoAWSCredsRule extends AbstractRestIT implements TestRule {
-
- @Autowired
- private Properties properties;
-
-
- public Statement apply( final Statement base, final Description description ) {
- return new Statement() {
- @Override
- public void evaluate() throws Throwable {
- String accessId;
- String secretKey;
- String bucketName;
- try {
- Map<String,Object> properties = getRemoteTestProperties();
- //TODO: GREY change this so that it checks for the properties, then if it doesn't have them, mark the tests as ignored.
- accessId = (String)System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
- secretKey = (String)System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
- bucketName =(String) properties.get( "usergrid.binary.bucketname" );
-
- if(accessId==null||secretKey==null||bucketName==null){
- throw new AwsPropertiesNotFoundException( "Access Keys" );
- }
- base.evaluate();
-
- }
- catch ( Throwable t ) {
-
- if ( !isMissingCredsException( t ) ) {
- throw t;
- }
-
- //do this so our test gets marked as ignored. Not pretty, but it works
- Assume.assumeTrue( false );
-
-
- }
- }
- };
- }
-
-
- private boolean isMissingCredsException( final Throwable t ) {
-
- if ( t instanceof AmazonClientException ) {
-
- final AmazonClientException ace = ( AmazonClientException ) t;
-
- if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
- "could not get aws secret key from system properties" ) ) {
- //swallow
- return true;
- }
- }
-
- if( t instanceof AwsPropertiesNotFoundException ){
- return true;
- }
-
- /**
- * Handle the multiple failure junit trace
- */
- if( t instanceof MultipleFailureException ){
- for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
- final boolean isMissingCreds = isMissingCredsException( failure );
-
- if(isMissingCreds){
- return true;
- }
- }
- }
- final Throwable cause = t.getCause();
-
- if ( cause == null ) {
- return false;
- }
-
-
- return isMissingCredsException( cause );
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java
new file mode 100644
index 0000000..8606e4e
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoAWSCredsRule.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.rest.applications.assets.rules;
+
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.SDKGlobalConfiguration;
+import org.apache.usergrid.rest.test.resource.AbstractRestIT;
+import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.FileInputStream;
+import java.util.Map;
+import java.util.Properties;
+
+
+/**
+ * Created in an attempt to mark no aws cred tests as ignored. Blocked by this issue
+ * https://github.com/junit-team/junit/issues/116
+ *
+ * Until then, simply marks as passed, which is a bit dangerous
+ */
+public class NoAWSCredsRule extends AbstractRestIT implements TestRule {
+
+ private static final Logger logger = LoggerFactory.getLogger( NoAWSCredsRule.class );
+
+
+ private Properties properties;
+
+
+ public Statement apply( final Statement base, final Description description ) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+
+
+ try {
+
+ final Map<String,Object> properties = getRemoteTestProperties();
+ final String bucketName = (String)properties.get( "usergrid.binary.bucketname" );
+
+
+ //TODO: GREY change this so that it checks for the properties, then if it doesn't have them, mark the tests as ignored.
+ final String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
+ final String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
+
+ if(accessId==null||secretKey==null||bucketName==null){
+ throw new AwsPropertiesNotFoundException( "Access Keys" );
+ }
+
+
+
+
+
+ base.evaluate();
+
+ }
+ catch ( Throwable t ) {
+
+ if ( !isMissingCredsException( t ) ) {
+ throw t;
+ }
+
+ //do this so our test gets marked as ignored. Not pretty, but it works
+ Assume.assumeTrue( false );
+
+
+ }
+ }
+ };
+ }
+
+
+ private boolean isMissingCredsException( final Throwable t ) {
+
+ if ( t instanceof AmazonClientException ) {
+
+ final AmazonClientException ace = ( AmazonClientException ) t;
+
+ if ( ace.getMessage().contains( "could not get aws access key" ) || ace.getMessage().contains(
+ "could not get aws secret key from system properties" ) ) {
+ //swallow
+ return true;
+ }
+ }
+
+ if( t instanceof AwsPropertiesNotFoundException ){
+ return true;
+ }
+
+ /**
+ * Handle the multiple failure junit trace
+ */
+ if( t instanceof MultipleFailureException ){
+ for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+ final boolean isMissingCreds = isMissingCredsException( failure );
+
+ if(isMissingCreds){
+ return true;
+ }
+ }
+ }
+ final Throwable cause = t.getCause();
+
+ if ( cause == null ) {
+ return false;
+ }
+
+
+ return isMissingCredsException( cause );
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java
new file mode 100644
index 0000000..5e3f66e
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/rules/NoGoogleCredsRule.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.rest.applications.assets.rules;
+
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.SDKGlobalConfiguration;
+import org.apache.usergrid.rest.test.resource.AbstractRestIT;
+import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.util.Map;
+import java.util.Properties;
+
+
+/**
+ * Created in an attempt to mark no Google cred tests as ignored. Blocked by this issue
+ * https://github.com/junit-team/junit/issues/116
+ *
+ * Until then, simply marks as passed, which is a bit dangerous
+ */
+public class NoGoogleCredsRule extends AbstractRestIT implements TestRule {
+
+ private static final Logger logger = LoggerFactory.getLogger( NoGoogleCredsRule.class );
+
+
+ @Autowired
+ private Properties properties;
+
+
+ public Statement apply( final Statement base, final Description description ) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+
+
+ try {
+
+ final String filename = System.getenv( "GOOGLE_APPLICATION_CREDENTIALS" );
+ logger.info("cred filename: {}", filename);
+ // if the file doesn't exist, an exception will be thrown
+ new FileInputStream(filename);
+
+ base.evaluate();
+
+ }
+ catch ( Throwable t ) {
+
+ if ( !isMissingCredsException( t ) ) {
+ throw t;
+ }
+
+ //do this so our test gets marked as ignored. Not pretty, but it works
+ Assume.assumeTrue( false );
+
+
+ }
+ }
+ };
+ }
+
+
+ private boolean isMissingCredsException( final Throwable t ) {
+
+ // either no filename was provided or the filename provided doesn't actually exist on the file system
+ if ( t instanceof FileNotFoundException || t instanceof NullPointerException ) {
+ return true;
+ }
+
+
+ /**
+ * Handle the multiple failure junit trace
+ */
+ if( t instanceof MultipleFailureException ){
+ for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+ final boolean isMissingCreds = isMissingCredsException( failure );
+
+ if(isMissingCreds){
+ return true;
+ }
+ }
+ }
+ final Throwable cause = t.getCause();
+
+ if ( cause == null ) {
+ return false;
+ }
+
+
+ return isMissingCredsException( cause );
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 8cd1756..b1df1b4 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -267,6 +267,12 @@
</dependency>
<dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-storage</artifactId>
+ <version>0.11.0-beta</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.jclouds</groupId>
<artifactId>jclouds-blobstore</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java
new file mode 100644
index 0000000..bd91dba
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/BinaryStoreFactory.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.services.assets;
+
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.services.assets.data.AWSBinaryStore;
+import org.apache.usergrid.services.assets.data.BinaryStore;
+import org.apache.usergrid.services.assets.data.GoogleBinaryStore;
+import org.apache.usergrid.services.assets.data.LocalFileBinaryStore;
+
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.Properties;
+
+import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
+
+
+public class BinaryStoreFactory {
+
+ public enum Provider{
+ local,aws,google
+ }
+
+ private EntityManagerFactory entityManagerFactory;
+ private Properties properties;
+ private String reposLocation;
+ private LocalFileBinaryStore localFileBinaryStore;
+ private AWSBinaryStore awsBinaryStore;
+ private GoogleBinaryStore googleCloudStorageBinaryStore;
+
+ public BinaryStoreFactory(Properties properties, EntityManagerFactory entityManagerFactory, String reposLocation) throws IOException, GeneralSecurityException {
+ this.properties = properties;
+ this.entityManagerFactory = entityManagerFactory;
+ this.reposLocation = reposLocation;
+ this.localFileBinaryStore = new LocalFileBinaryStore(properties, entityManagerFactory, reposLocation);
+ this.awsBinaryStore = new AWSBinaryStore(properties, entityManagerFactory, reposLocation);
+ this.googleCloudStorageBinaryStore = new GoogleBinaryStore(properties, entityManagerFactory, reposLocation);
+ }
+
+ public synchronized BinaryStore getBinaryStore(String provider) throws IOException, GeneralSecurityException {
+
+ provider = provider != null? provider.toLowerCase(): "";
+
+ if( provider.isEmpty() ){
+
+ if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( Provider.local.name() )){
+ return localFileBinaryStore;
+ } else if (properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( Provider.google.name() )){
+ return googleCloudStorageBinaryStore;
+ } else{
+ return awsBinaryStore;
+ }
+ }
+
+ if ( provider.equals(Provider.local.name())){
+ return localFileBinaryStore;
+ }
+ if ( provider.equals(Provider.google.name())){
+ return googleCloudStorageBinaryStore;
+ }
+ if( provider.equals(Provider.aws.name())){
+ return awsBinaryStore;
+ }
+
+ // this for backwards compatibility because historically anything other than "local" meant AWS
+ return awsBinaryStore;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java
new file mode 100644
index 0000000..14da2e1
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AWSBinaryStore.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.services.assets.data;
+
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
+import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
+import org.apache.usergrid.utils.StringUtils;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
+import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
+import com.amazonaws.services.s3.model.MultipartUpload;
+import com.amazonaws.services.s3.model.MultipartUploadListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PartETag;
+import com.amazonaws.services.s3.model.PutObjectResult;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.google.common.primitives.Ints;
+import java.io.ByteArrayInputStream;
+import java.io.FileInputStream;
+import java.io.PushbackInputStream;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.commons.codec.binary.Base64;
+
+
+public class AWSBinaryStore implements BinaryStore {
+
+ private static final Logger logger = LoggerFactory.getLogger(AWSBinaryStore.class );
+ private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 );
+
+ private AmazonS3 s3Client;
+ private String accessId;
+ private String secretKey;
+ private String bucketName;
+ private String regionName;
+
+ private EntityManagerFactory emf;
+ private Properties properties;
+ private String reposLocation;
+
+ public AWSBinaryStore(Properties properties,
+ EntityManagerFactory entityManagerFactory,
+ String reposLocation) {
+ this.properties = properties;
+ this.emf = entityManagerFactory;
+ this.reposLocation = reposLocation;
+
+ }
+
+ //TODO: GREY rework how the s3 client works because currently it handles initlization and returning of the client
+ //ideally it should only do one. and the client should be initlized at the beginning of the run.
+ private AmazonS3 getS3Client() throws Exception{
+
+ this.bucketName = properties.getProperty( "usergrid.binary.bucketname" );
+ if(bucketName == null){
+ logger.error( "usergrid.binary.bucketname not properly set so amazon bucket is null" );
+ throw new AwsPropertiesNotFoundException( "usergrid.binary.bucketname" );
+
+ }
+
+ final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
+ AWSCredentials credentials = ugProvider.getCredentials();
+ ClientConfiguration clientConfig = new ClientConfiguration();
+ clientConfig.setProtocol(Protocol.HTTP);
+
+ s3Client = new AmazonS3Client(credentials, clientConfig);
+ if(regionName != null)
+ s3Client.setRegion( Region.getRegion(Regions.fromName(regionName)) );
+
+ return s3Client;
+ }
+
+ @Override
+ public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws Exception {
+
+ String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB );
+
+ byte[] data = baos.toByteArray();
+
+ InputStream awsInputStream = new ByteArrayInputStream(data);
+
+ final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
+ fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
+
+ String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
+
+ Boolean overSizeLimit = false;
+
+ EntityManager em = emf.getEntityManager( appId );
+
+
+ if ( written < FIVE_MB ) { // total smaller than 5mb
+
+ ObjectMetadata om = new ObjectMetadata();
+ om.setContentLength(written);
+ om.setContentType( mimeType );
+ PutObjectResult result = null;
+ result = getS3Client().putObject( bucketName, uploadFileName, awsInputStream, om );
+
+ String md5sum = Hex.encodeHexString( Base64.decodeBase64(result.getContentMd5()) );
+ String eTag = result.getETag();
+
+ fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
+
+ if(md5sum != null)
+ fileMetadata.put( AssetUtils.CHECKSUM, md5sum );
+ fileMetadata.put( AssetUtils.E_TAG, eTag );
+
+ em.update( entity );
+
+ }
+ else { // bigger than 5mb... dump 5 mb tmp files and upload from them
+ written = 0; //reset written to 0, we still haven't wrote anything in fact
+ int partNumber = 1;
+ int firstByte = 0;
+ Boolean isFirstChunck = true;
+ List<PartETag> partETags = new ArrayList<PartETag>();
+
+
+ //get the s3 client in order to initialize the multipart request
+ getS3Client();
+ InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest( bucketName, uploadFileName );
+ InitiateMultipartUploadResult initResponse = getS3Client().initiateMultipartUpload( initRequest );
+
+
+ InputStream firstChunck = new ByteArrayInputStream(data);
+ PushbackInputStream chunckableInputStream = new PushbackInputStream(inputStream, 1);
+
+ // determine max size file allowed, default to 50mb
+ long maxSizeBytes = 50 * FileUtils.ONE_MB;
+ String maxSizeMbString = properties.getProperty( "usergrid.binary.max-size-mb", "50" );
+ if ( StringUtils.isNumeric( maxSizeMbString )) {
+ maxSizeBytes = Long.parseLong( maxSizeMbString ) * FileUtils.ONE_MB;
+ }
+
+ // always allow files up to 5mb
+ if (maxSizeBytes < 5 * FileUtils.ONE_MB ) {
+ maxSizeBytes = 5 * FileUtils.ONE_MB;
+ }
+
+ while (-1 != (firstByte = chunckableInputStream.read())) {
+ long partSize = 0;
+ chunckableInputStream.unread(firstByte);
+ File tempFile = File.createTempFile( entity.getUuid().toString().concat("-part").concat(String.valueOf(partNumber)), "tmp" );
+
+ tempFile.deleteOnExit();
+ OutputStream os = null;
+ try {
+ os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath() ) );
+
+ if(isFirstChunck == true) {
+ partSize = IOUtils.copyLarge( firstChunck, os, 0, ( FIVE_MB ) );
+ isFirstChunck = false;
+ }
+ else {
+ partSize = IOUtils.copyLarge( chunckableInputStream, os, 0, ( FIVE_MB ) );
+ }
+ written += partSize;
+
+ if(written> maxSizeBytes){
+ overSizeLimit = true;
+ logger.error( "OVERSIZED FILE ({}). STARTING ABORT", written );
+ break;
+ //set flag here and break out of loop to run abort
+ }
+ }
+ finally {
+ IOUtils.closeQuietly( os );
+ }
+
+ FileInputStream chunk = new FileInputStream(tempFile);
+
+ Boolean isLastPart = -1 == (firstByte = chunckableInputStream.read());
+ if(!isLastPart)
+ chunckableInputStream.unread(firstByte);
+
+ UploadPartRequest uploadRequest = new UploadPartRequest().withUploadId(initResponse.getUploadId())
+ .withBucketName(bucketName)
+ .withKey(uploadFileName)
+ .withInputStream(chunk)
+ .withPartNumber(partNumber)
+ .withPartSize(partSize)
+ .withLastPart(isLastPart);
+ partETags.add( getS3Client().uploadPart(uploadRequest).getPartETag() );
+ partNumber++;
+ }
+
+ //check for flag here then abort.
+ if(overSizeLimit) {
+
+ AbortMultipartUploadRequest abortRequest =
+ new AbortMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId() );
+
+ ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest( bucketName );
+
+ MultipartUploadListing listResult = getS3Client().listMultipartUploads( listRequest );
+
+ //upadte the entity with the error.
+ try {
+ logger.error("starting update of entity due to oversized asset");
+ fileMetadata.put( "error", "Asset size is larger than max size of " + maxSizeBytes );
+ em.update( entity );
+ }
+ catch ( Exception e ) {
+ logger.error( "Error updating entity with error message", e );
+ }
+
+ int timesIterated = 20;
+ //loop and abort all the multipart uploads
+ while ( listResult.getMultipartUploads().size()!=0 && timesIterated > 0) {
+
+ getS3Client().abortMultipartUpload( abortRequest );
+ Thread.sleep( 1000 );
+ timesIterated--;
+ listResult = getS3Client().listMultipartUploads( listRequest );
+ if (logger.isDebugEnabled()) {
+ logger.debug("Files that haven't been aborted are: {}", listResult.getMultipartUploads().listIterator().toString());
+ }
+
+ }
+ if ( timesIterated == 0 ){
+ logger.error( "Files parts that couldn't be aborted in 20 seconds are:" );
+ Iterator<MultipartUpload> multipartUploadIterator = listResult.getMultipartUploads().iterator();
+ while(multipartUploadIterator.hasNext()){
+ logger.error( multipartUploadIterator.next().getKey() );
+ }
+ }
+ }
+ else {
+ CompleteMultipartUploadRequest request =
+ new CompleteMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId(),
+ partETags );
+ CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request );
+ fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
+ fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() );
+ em.update( entity );
+ }
+ }
+ }
+
+
+ @Override
+ public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception {
+
+ S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) );
+
+ byte data[] = null;
+
+ if ( offset == 0 && length == FIVE_MB ) {
+ return object.getObjectContent();
+ }
+ else {
+ object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length));
+ }
+
+ return new ByteArrayInputStream(data);
+ }
+
+
+ @Override
+ public InputStream read( UUID appId, Entity entity ) throws Exception {
+ return read( appId, entity, 0, FIVE_MB );
+ }
+
+
+ @Override
+ public void delete( UUID appId, Entity entity ) throws Exception {
+ getS3Client().deleteObject(new DeleteObjectRequest(bucketName, AssetUtils.buildAssetKey( appId, entity )));
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bafd4627/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
deleted file mode 100644
index 5f2c041..0000000
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
+++ /dev/null
@@ -1,317 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.services.assets.data;
-
-
-import java.io.BufferedOutputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.queue.impl.UsergridAwsCredentialsProvider;
-import org.apache.usergrid.services.exceptions.AwsPropertiesNotFoundException;
-import org.apache.usergrid.utils.StringUtils;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.regions.Region;
-import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
-import com.amazonaws.services.s3.model.DeleteObjectRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
-import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
-import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
-import com.amazonaws.services.s3.model.MultipartUpload;
-import com.amazonaws.services.s3.model.MultipartUploadListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
-import com.amazonaws.services.s3.model.PutObjectResult;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.UploadPartRequest;
-import com.google.common.primitives.Ints;
-import java.io.ByteArrayInputStream;
-import java.io.FileInputStream;
-import java.io.PushbackInputStream;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.commons.codec.binary.Base64;
-
-
-public class AwsSdkS3BinaryStore implements BinaryStore {
-
- private static final Logger logger = LoggerFactory.getLogger(AwsSdkS3BinaryStore.class );
- private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 );
-
- private AmazonS3 s3Client;
- private String accessId;
- private String secretKey;
- private String bucketName;
- private String regionName;
-
- @Autowired
- private EntityManagerFactory emf;
-
- @Autowired
- private Properties properties;
-
- public AwsSdkS3BinaryStore( ) {
- }
-
- //TODO: GREY rework how the s3 client works because currently it handles initlization and returning of the client
- //ideally it should only do one. and the client should be initlized at the beginning of the run.
- private AmazonS3 getS3Client() throws Exception{
-
- this.bucketName = properties.getProperty( "usergrid.binary.bucketname" );
- if(bucketName == null){
- logger.error( "usergrid.binary.bucketname not properly set so amazon bucket is null" );
- throw new AwsPropertiesNotFoundException( "usergrid.binary.bucketname" );
-
- }
-
- final UsergridAwsCredentialsProvider ugProvider = new UsergridAwsCredentialsProvider();
- AWSCredentials credentials = ugProvider.getCredentials();
- ClientConfiguration clientConfig = new ClientConfiguration();
- clientConfig.setProtocol(Protocol.HTTP);
-
- s3Client = new AmazonS3Client(credentials, clientConfig);
- if(regionName != null)
- s3Client.setRegion( Region.getRegion(Regions.fromName(regionName)) );
-
- return s3Client;
- }
-
- @Override
- public void write( final UUID appId, final Entity entity, InputStream inputStream ) throws Exception {
-
- String uploadFileName = AssetUtils.buildAssetKey( appId, entity );
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- long written = IOUtils.copyLarge( inputStream, baos, 0, FIVE_MB );
-
- byte[] data = baos.toByteArray();
-
- InputStream awsInputStream = new ByteArrayInputStream(data);
-
- final Map<String, Object> fileMetadata = AssetUtils.getFileMetadata( entity );
- fileMetadata.put( AssetUtils.LAST_MODIFIED, System.currentTimeMillis() );
-
- String mimeType = AssetMimeHandler.get().getMimeType( entity, data );
-
- Boolean overSizeLimit = false;
-
- EntityManager em = emf.getEntityManager( appId );
-
-
- if ( written < FIVE_MB ) { // total smaller than 5mb
-
- ObjectMetadata om = new ObjectMetadata();
- om.setContentLength(written);
- om.setContentType( mimeType );
- PutObjectResult result = null;
- result = getS3Client().putObject( bucketName, uploadFileName, awsInputStream, om );
-
- String md5sum = Hex.encodeHexString( Base64.decodeBase64(result.getContentMd5()) );
- String eTag = result.getETag();
-
- fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
-
- if(md5sum != null)
- fileMetadata.put( AssetUtils.CHECKSUM, md5sum );
- fileMetadata.put( AssetUtils.E_TAG, eTag );
-
- em.update( entity );
-
- }
- else { // bigger than 5mb... dump 5 mb tmp files and upload from them
- written = 0; //reset written to 0, we still haven't wrote anything in fact
- int partNumber = 1;
- int firstByte = 0;
- Boolean isFirstChunck = true;
- List<PartETag> partETags = new ArrayList<PartETag>();
-
-
- //get the s3 client in order to initialize the multipart request
- getS3Client();
- InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest( bucketName, uploadFileName );
- InitiateMultipartUploadResult initResponse = getS3Client().initiateMultipartUpload( initRequest );
-
-
- InputStream firstChunck = new ByteArrayInputStream(data);
- PushbackInputStream chunckableInputStream = new PushbackInputStream(inputStream, 1);
-
- // determine max size file allowed, default to 50mb
- long maxSizeBytes = 50 * FileUtils.ONE_MB;
- String maxSizeMbString = properties.getProperty( "usergrid.binary.max-size-mb", "50" );
- if ( StringUtils.isNumeric( maxSizeMbString )) {
- maxSizeBytes = Long.parseLong( maxSizeMbString ) * FileUtils.ONE_MB;
- }
-
- // always allow files up to 5mb
- if (maxSizeBytes < 5 * FileUtils.ONE_MB ) {
- maxSizeBytes = 5 * FileUtils.ONE_MB;
- }
-
- while (-1 != (firstByte = chunckableInputStream.read())) {
- long partSize = 0;
- chunckableInputStream.unread(firstByte);
- File tempFile = File.createTempFile( entity.getUuid().toString().concat("-part").concat(String.valueOf(partNumber)), "tmp" );
-
- tempFile.deleteOnExit();
- OutputStream os = null;
- try {
- os = new BufferedOutputStream( new FileOutputStream( tempFile.getAbsolutePath() ) );
-
- if(isFirstChunck == true) {
- partSize = IOUtils.copyLarge( firstChunck, os, 0, ( FIVE_MB ) );
- isFirstChunck = false;
- }
- else {
- partSize = IOUtils.copyLarge( chunckableInputStream, os, 0, ( FIVE_MB ) );
- }
- written += partSize;
-
- if(written> maxSizeBytes){
- overSizeLimit = true;
- logger.error( "OVERSIZED FILE ({}). STARTING ABORT", written );
- break;
- //set flag here and break out of loop to run abort
- }
- }
- finally {
- IOUtils.closeQuietly( os );
- }
-
- FileInputStream chunk = new FileInputStream(tempFile);
-
- Boolean isLastPart = -1 == (firstByte = chunckableInputStream.read());
- if(!isLastPart)
- chunckableInputStream.unread(firstByte);
-
- UploadPartRequest uploadRequest = new UploadPartRequest().withUploadId(initResponse.getUploadId())
- .withBucketName(bucketName)
- .withKey(uploadFileName)
- .withInputStream(chunk)
- .withPartNumber(partNumber)
- .withPartSize(partSize)
- .withLastPart(isLastPart);
- partETags.add( getS3Client().uploadPart(uploadRequest).getPartETag() );
- partNumber++;
- }
-
- //check for flag here then abort.
- if(overSizeLimit) {
-
- AbortMultipartUploadRequest abortRequest =
- new AbortMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId() );
-
- ListMultipartUploadsRequest listRequest = new ListMultipartUploadsRequest( bucketName );
-
- MultipartUploadListing listResult = getS3Client().listMultipartUploads( listRequest );
-
- //upadte the entity with the error.
- try {
- logger.error("starting update of entity due to oversized asset");
- fileMetadata.put( "error", "Asset size is larger than max size of " + maxSizeBytes );
- em.update( entity );
- }
- catch ( Exception e ) {
- logger.error( "Error updating entity with error message", e );
- }
-
- int timesIterated = 20;
- //loop and abort all the multipart uploads
- while ( listResult.getMultipartUploads().size()!=0 && timesIterated > 0) {
-
- getS3Client().abortMultipartUpload( abortRequest );
- Thread.sleep( 1000 );
- timesIterated--;
- listResult = getS3Client().listMultipartUploads( listRequest );
- if (logger.isDebugEnabled()) {
- logger.debug("Files that haven't been aborted are: {}", listResult.getMultipartUploads().listIterator().toString());
- }
-
- }
- if ( timesIterated == 0 ){
- logger.error( "Files parts that couldn't be aborted in 20 seconds are:" );
- Iterator<MultipartUpload> multipartUploadIterator = listResult.getMultipartUploads().iterator();
- while(multipartUploadIterator.hasNext()){
- logger.error( multipartUploadIterator.next().getKey() );
- }
- }
- }
- else {
- CompleteMultipartUploadRequest request =
- new CompleteMultipartUploadRequest( bucketName, uploadFileName, initResponse.getUploadId(),
- partETags );
- CompleteMultipartUploadResult amazonResult = getS3Client().completeMultipartUpload( request );
- fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
- fileMetadata.put( AssetUtils.E_TAG, amazonResult.getETag() );
- em.update( entity );
- }
- }
- }
-
-
- @Override
- public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception {
-
- S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) );
-
- byte data[] = null;
-
- if ( offset == 0 && length == FIVE_MB ) {
- return object.getObjectContent();
- }
- else {
- object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length));
- }
-
- return new ByteArrayInputStream(data);
- }
-
-
- @Override
- public InputStream read( UUID appId, Entity entity ) throws Exception {
- return read( appId, entity, 0, FIVE_MB );
- }
-
-
- @Override
- public void delete( UUID appId, Entity entity ) throws Exception {
- getS3Client().deleteObject(new DeleteObjectRequest(bucketName, AssetUtils.buildAssetKey( appId, entity )));
- }
-}