You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/07/23 21:08:11 UTC
incubator-usergrid git commit: Adding properties check for the asset
service endpoints for testing purposes only changed resources to include both
local and was endpoint . Increased timeout for loading large files into s3.
Removed binary store informatio
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-869 c8d616a90 -> 050aa0a03
Adding properties check for the asset service endpoints for testing purposes only
changed resources to include both local and was endpoint .
Increased timeout for loading large files into s3.
Removed binary store information from test context as regular context supports both deployments
Added log messages to AwsSdkBinaryStore
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/050aa0a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/050aa0a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/050aa0a0
Branch: refs/heads/USERGRID-869
Commit: 050aa0a035fffbb202a45251e48b542d3053a655
Parents: c8d616a
Author: GERey <gr...@apigee.com>
Authored: Thu Jul 23 12:08:08 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Thu Jul 23 12:08:08 2015 -0700
----------------------------------------------------------------------
.../rest/applications/ServiceResource.java | 26 +++++++++++++++++
.../main/resources/usergrid-rest-context.xml | 14 ++++++++-
.../applications/assets/AwsAssetResourceIT.java | 30 ++++++++++----------
.../test/resources/usergrid-test-context.xml | 8 ------
.../assets/data/AwsSdkS3BinaryStore.java | 6 ++--
.../test/resources/usergrid-test-context.xml | 2 +-
6 files changed, 59 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index fc729d4..bd5d89f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -83,6 +83,8 @@ import com.sun.jersey.multipart.FormDataMultiPart;
import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE;
+import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
+
@Component
@Scope("prototype")
@@ -582,6 +584,14 @@ public class ServiceResource extends AbstractContextResource {
private JSONWithPadding executeMultiPart( UriInfo ui, String callback, FormDataMultiPart multiPart,
ServiceAction serviceAction ) throws Exception {
+ //needed for testing
+ if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
+ this.binaryStore = localFileBinaryStore;
+ }
+ else{
+ this.binaryStore = awsSdkS3BinaryStore;
+ }
+
// collect form data values
List<BodyPart> bodyParts = multiPart.getBodyParts();
HashMap<String, Object> data = new HashMap<String, Object>();
@@ -638,6 +648,14 @@ public class ServiceResource extends AbstractContextResource {
@Consumes(MediaType.APPLICATION_OCTET_STREAM)
public Response uploadDataStream( @Context UriInfo ui, InputStream uploadedInputStream ) throws Exception {
+ //needed for testing
+ if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
+ this.binaryStore = localFileBinaryStore;
+ }
+ else{
+ this.binaryStore = awsSdkS3BinaryStore;
+ }
+
ApiResponse response = createApiResponse();
response.setAction( "get" );
response.setApplication( services.getApplication() );
@@ -662,6 +680,14 @@ public class ServiceResource extends AbstractContextResource {
LOG.debug( "ServiceResource.executeStreamGet" );
+ //Needed for testing
+ if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
+ this.binaryStore = localFileBinaryStore;
+ }
+ else{
+ this.binaryStore = awsSdkS3BinaryStore;
+ }
+
ApiResponse response = createApiResponse();
response.setAction( "get" );
response.setApplication( services.getApplication() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/main/resources/usergrid-rest-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/resources/usergrid-rest-context.xml b/stack/rest/src/main/resources/usergrid-rest-context.xml
index 16f4541..438cd1c 100644
--- a/stack/rest/src/main/resources/usergrid-rest-context.xml
+++ b/stack/rest/src/main/resources/usergrid-rest-context.xml
@@ -65,8 +65,20 @@
<property name="queueCapacity" value="25" />
</bean>
- <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
+ <bean id="serviceResource"
+ class="org.apache.usergrid.rest.applications.ServiceResource" scope="prototype">
+ <property name="binaryStore" value="${usergrid.binary.uploader}"/>
+ </bean>
+
+ <bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
<property name="reposLocation" value="${usergrid.temp.files}"/>
</bean>
+ <bean id="awsS3BinaryStore" class="org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore">
+ <constructor-arg name="accessId" value="xx" />
+ <constructor-arg name="secretKey" value="xx" />
+ <constructor-arg name="bucketName" value="xx" />
+ </bean>
+
+
</beans>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
index e512224..2914371 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
@@ -50,6 +50,8 @@ import org.apache.usergrid.setup.ConcurrentProcessSingleton;
import com.sun.jersey.multipart.FormDataMultiPart;
+import net.jcip.annotations.NotThreadSafe;
+
import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_SYSADMIN_EMAIL;
import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
import static org.apache.usergrid.utils.MapUtils.hashMap;
@@ -59,18 +61,18 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
-
+@NotThreadSafe
public class AwsAssetResourceIT extends AbstractRestIT {
private String access_token;
private Map<String, Object> originalProperties;
private Logger LOG = LoggerFactory.getLogger( AwsAssetResourceIT.class );
-// /**
-// * Mark tests as ignored if no AWS creds are present
-// */
-// @Rule
-// public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
+ /**
+ * Mark tests as ignored if no AWS creds are present
+ */
+ @Rule
+ public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
@Before
public void setup(){
@@ -230,7 +232,7 @@ public class AwsAssetResourceIT extends AbstractRestIT {
ApiResponse postResponse = pathResource( getOrgAppPath( "foos" ) ).post( form );
UUID assetId = postResponse.getEntities().get(0).getUuid();
LOG.info( "Waiting for upload to finish..." );
- Thread.sleep( 2000 );
+ Thread.sleep( 5000 );
// check that entire file was uploaded
@@ -251,10 +253,7 @@ public class AwsAssetResourceIT extends AbstractRestIT {
this.refreshIndex();
// set max file size down to 6mb
-
- Map<String, String> props = new HashMap<String, String>();
- props.put( "usergrid.binary.max-size-mb", "6" );
- pathResource( "testproperties" ).post( props );
+ setTestProperty( "usergrid.binary.max-size-mb","6" );
try {
@@ -267,20 +266,21 @@ public class AwsAssetResourceIT extends AbstractRestIT {
String errorMessage = null;
LOG.info( "Waiting for upload to finish..." );
- Thread.sleep( 2000 );
+ Thread.sleep( 1000 );
// attempt to get asset entity, it should contain error
+ refreshIndex();
ApiResponse getResponse = pathResource( getOrgAppPath( "bars/" +assetId ) ).get( ApiResponse.class );
Map<String, Object> fileMetadata = (Map<String, Object>)getResponse.getEntities().get(0).get("file-metadata");
+ assertNotNull( fileMetadata );
+ assertNotNull( fileMetadata.get( "error" ) );
assertTrue( fileMetadata.get( "error" ).toString().startsWith( "Asset size " ) );
} finally {
// set max upload size back to default 25mb
-
- props.put( "usergrid.binary.max-size-mb", "25" );
- pathResource( "testproperties" ).post( props );
+ setTestProperties( originalProperties );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-test-context.xml b/stack/rest/src/test/resources/usergrid-test-context.xml
index 6b15b7d..a9fc5a5 100644
--- a/stack/rest/src/test/resources/usergrid-test-context.xml
+++ b/stack/rest/src/test/resources/usergrid-test-context.xml
@@ -44,14 +44,6 @@
<property name="traceEnabled" value="false"/>
</bean>
- <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore"/>
-
- <!--<bean id="binaryStore" class="org.apache.usergrid.services.assets.data.S3BinaryStore">-->
- <!--<constructor-arg name="accessId" value="xx" />-->
- <!--<constructor-arg name="secretKey" value="xx" />-->
- <!--<constructor-arg name="bucketName" value="xx" />-->
- <!--</bean>-->
-
<bean id="setup" class="org.apache.usergrid.corepersistence.CpSetup">
<constructor-arg ref="entityManagerFactory"/>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
index d34a273..e492f42 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
@@ -73,7 +73,7 @@ import org.apache.commons.codec.binary.Base64;
public class AwsSdkS3BinaryStore implements BinaryStore {
- private static final Logger LOG = LoggerFactory.getLogger(AwsSdkS3BinaryStore.class );
+ private static final Logger logger = LoggerFactory.getLogger(AwsSdkS3BinaryStore.class );
private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 );
private AmazonS3 s3Client;
@@ -197,6 +197,7 @@ public class AwsSdkS3BinaryStore implements BinaryStore {
if(written> maxSizeBytes){
overSizeLimit = true;
+ logger.error( "OVERSIZED FILE. STARTING ABORT" );
break;
//set flag here and break out of loop to run abort
}
@@ -235,11 +236,12 @@ public class AwsSdkS3BinaryStore implements BinaryStore {
//upadte the entity with the error.
try {
+ logger.error("starting update of entity due to oversized asset");
fileMetadata.put( "error", "Asset size is larger than max size of " + maxSizeBytes );
em.update( entity );
}
catch ( Exception e ) {
- LOG.error( "Error updating entity with error message", e );
+ logger.error( "Error updating entity with error message", e );
}
//loop and abort all the multipart uploads
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/services/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-test-context.xml b/stack/services/src/test/resources/usergrid-test-context.xml
index f6d750d..ec1e2d2 100644
--- a/stack/services/src/test/resources/usergrid-test-context.xml
+++ b/stack/services/src/test/resources/usergrid-test-context.xml
@@ -51,7 +51,7 @@
<constructor-arg ref="cassandraCluster"/>
</bean>
- <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
+ <bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
<property name="reposLocation" value="${usergrid.temp.files}"/>
</bean>