You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/07/27 15:45:38 UTC
[04/11] incubator-usergrid git commit: Adding properties check for
the asset service endpoints for testing purposes only changed resources to
include both local and was endpoint . Increased timeout for loading large
files into s3. Removed binary store in
Adding properties check for the asset service endpoints for testing purposes only
changed resources to include both local and was endpoint .
Increased timeout for loading large files into s3.
Removed binary store information from test context as regular context supports both deployments
Added log messages to AwsSdkBinaryStore
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/050aa0a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/050aa0a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/050aa0a0
Branch: refs/heads/two-dot-o-dev
Commit: 050aa0a035fffbb202a45251e48b542d3053a655
Parents: c8d616a
Author: GERey <gr...@apigee.com>
Authored: Thu Jul 23 12:08:08 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Thu Jul 23 12:08:08 2015 -0700
----------------------------------------------------------------------
.../rest/applications/ServiceResource.java | 26 +++++++++++++++++
.../main/resources/usergrid-rest-context.xml | 14 ++++++++-
.../applications/assets/AwsAssetResourceIT.java | 30 ++++++++++----------
.../test/resources/usergrid-test-context.xml | 8 ------
.../assets/data/AwsSdkS3BinaryStore.java | 6 ++--
.../test/resources/usergrid-test-context.xml | 2 +-
6 files changed, 59 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index fc729d4..bd5d89f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -83,6 +83,8 @@ import com.sun.jersey.multipart.FormDataMultiPart;
import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE;
+import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
+
@Component
@Scope("prototype")
@@ -582,6 +584,14 @@ public class ServiceResource extends AbstractContextResource {
private JSONWithPadding executeMultiPart( UriInfo ui, String callback, FormDataMultiPart multiPart,
ServiceAction serviceAction ) throws Exception {
+ //needed for testing
+ if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
+ this.binaryStore = localFileBinaryStore;
+ }
+ else{
+ this.binaryStore = awsSdkS3BinaryStore;
+ }
+
// collect form data values
List<BodyPart> bodyParts = multiPart.getBodyParts();
HashMap<String, Object> data = new HashMap<String, Object>();
@@ -638,6 +648,14 @@ public class ServiceResource extends AbstractContextResource {
@Consumes(MediaType.APPLICATION_OCTET_STREAM)
public Response uploadDataStream( @Context UriInfo ui, InputStream uploadedInputStream ) throws Exception {
+ //needed for testing
+ if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
+ this.binaryStore = localFileBinaryStore;
+ }
+ else{
+ this.binaryStore = awsSdkS3BinaryStore;
+ }
+
ApiResponse response = createApiResponse();
response.setAction( "get" );
response.setApplication( services.getApplication() );
@@ -662,6 +680,14 @@ public class ServiceResource extends AbstractContextResource {
LOG.debug( "ServiceResource.executeStreamGet" );
+ //Needed for testing
+ if(properties.getProperty( PROPERTIES_USERGRID_BINARY_UPLOADER ).equals( "local" )){
+ this.binaryStore = localFileBinaryStore;
+ }
+ else{
+ this.binaryStore = awsSdkS3BinaryStore;
+ }
+
ApiResponse response = createApiResponse();
response.setAction( "get" );
response.setApplication( services.getApplication() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/main/resources/usergrid-rest-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/resources/usergrid-rest-context.xml b/stack/rest/src/main/resources/usergrid-rest-context.xml
index 16f4541..438cd1c 100644
--- a/stack/rest/src/main/resources/usergrid-rest-context.xml
+++ b/stack/rest/src/main/resources/usergrid-rest-context.xml
@@ -65,8 +65,20 @@
<property name="queueCapacity" value="25" />
</bean>
- <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
+ <bean id="serviceResource"
+ class="org.apache.usergrid.rest.applications.ServiceResource" scope="prototype">
+ <property name="binaryStore" value="${usergrid.binary.uploader}"/>
+ </bean>
+
+ <bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
<property name="reposLocation" value="${usergrid.temp.files}"/>
</bean>
+ <bean id="awsS3BinaryStore" class="org.apache.usergrid.services.assets.data.AwsSdkS3BinaryStore">
+ <constructor-arg name="accessId" value="xx" />
+ <constructor-arg name="secretKey" value="xx" />
+ <constructor-arg name="bucketName" value="xx" />
+ </bean>
+
+
</beans>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
index e512224..2914371 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/assets/AwsAssetResourceIT.java
@@ -50,6 +50,8 @@ import org.apache.usergrid.setup.ConcurrentProcessSingleton;
import com.sun.jersey.multipart.FormDataMultiPart;
+import net.jcip.annotations.NotThreadSafe;
+
import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_SYSADMIN_EMAIL;
import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_USERGRID_BINARY_UPLOADER;
import static org.apache.usergrid.utils.MapUtils.hashMap;
@@ -59,18 +61,18 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
-
+@NotThreadSafe
public class AwsAssetResourceIT extends AbstractRestIT {
private String access_token;
private Map<String, Object> originalProperties;
private Logger LOG = LoggerFactory.getLogger( AwsAssetResourceIT.class );
-// /**
-// * Mark tests as ignored if no AWS creds are present
-// */
-// @Rule
-// public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
+ /**
+ * Mark tests as ignored if no AWS creds are present
+ */
+ @Rule
+ public NoAWSCredsRule awsCredsRule = new NoAWSCredsRule();
@Before
public void setup(){
@@ -230,7 +232,7 @@ public class AwsAssetResourceIT extends AbstractRestIT {
ApiResponse postResponse = pathResource( getOrgAppPath( "foos" ) ).post( form );
UUID assetId = postResponse.getEntities().get(0).getUuid();
LOG.info( "Waiting for upload to finish..." );
- Thread.sleep( 2000 );
+ Thread.sleep( 5000 );
// check that entire file was uploaded
@@ -251,10 +253,7 @@ public class AwsAssetResourceIT extends AbstractRestIT {
this.refreshIndex();
// set max file size down to 6mb
-
- Map<String, String> props = new HashMap<String, String>();
- props.put( "usergrid.binary.max-size-mb", "6" );
- pathResource( "testproperties" ).post( props );
+ setTestProperty( "usergrid.binary.max-size-mb","6" );
try {
@@ -267,20 +266,21 @@ public class AwsAssetResourceIT extends AbstractRestIT {
String errorMessage = null;
LOG.info( "Waiting for upload to finish..." );
- Thread.sleep( 2000 );
+ Thread.sleep( 1000 );
// attempt to get asset entity, it should contain error
+ refreshIndex();
ApiResponse getResponse = pathResource( getOrgAppPath( "bars/" +assetId ) ).get( ApiResponse.class );
Map<String, Object> fileMetadata = (Map<String, Object>)getResponse.getEntities().get(0).get("file-metadata");
+ assertNotNull( fileMetadata );
+ assertNotNull( fileMetadata.get( "error" ) );
assertTrue( fileMetadata.get( "error" ).toString().startsWith( "Asset size " ) );
} finally {
// set max upload size back to default 25mb
-
- props.put( "usergrid.binary.max-size-mb", "25" );
- pathResource( "testproperties" ).post( props );
+ setTestProperties( originalProperties );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/rest/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-test-context.xml b/stack/rest/src/test/resources/usergrid-test-context.xml
index 6b15b7d..a9fc5a5 100644
--- a/stack/rest/src/test/resources/usergrid-test-context.xml
+++ b/stack/rest/src/test/resources/usergrid-test-context.xml
@@ -44,14 +44,6 @@
<property name="traceEnabled" value="false"/>
</bean>
- <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore"/>
-
- <!--<bean id="binaryStore" class="org.apache.usergrid.services.assets.data.S3BinaryStore">-->
- <!--<constructor-arg name="accessId" value="xx" />-->
- <!--<constructor-arg name="secretKey" value="xx" />-->
- <!--<constructor-arg name="bucketName" value="xx" />-->
- <!--</bean>-->
-
<bean id="setup" class="org.apache.usergrid.corepersistence.CpSetup">
<constructor-arg ref="entityManagerFactory"/>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
index d34a273..e492f42 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/AwsSdkS3BinaryStore.java
@@ -73,7 +73,7 @@ import org.apache.commons.codec.binary.Base64;
public class AwsSdkS3BinaryStore implements BinaryStore {
- private static final Logger LOG = LoggerFactory.getLogger(AwsSdkS3BinaryStore.class );
+ private static final Logger logger = LoggerFactory.getLogger(AwsSdkS3BinaryStore.class );
private static final long FIVE_MB = ( FileUtils.ONE_MB * 5 );
private AmazonS3 s3Client;
@@ -197,6 +197,7 @@ public class AwsSdkS3BinaryStore implements BinaryStore {
if(written> maxSizeBytes){
overSizeLimit = true;
+ logger.error( "OVERSIZED FILE. STARTING ABORT" );
break;
//set flag here and break out of loop to run abort
}
@@ -235,11 +236,12 @@ public class AwsSdkS3BinaryStore implements BinaryStore {
//upadte the entity with the error.
try {
+ logger.error("starting update of entity due to oversized asset");
fileMetadata.put( "error", "Asset size is larger than max size of " + maxSizeBytes );
em.update( entity );
}
catch ( Exception e ) {
- LOG.error( "Error updating entity with error message", e );
+ logger.error( "Error updating entity with error message", e );
}
//loop and abort all the multipart uploads
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/050aa0a0/stack/services/src/test/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-test-context.xml b/stack/services/src/test/resources/usergrid-test-context.xml
index f6d750d..ec1e2d2 100644
--- a/stack/services/src/test/resources/usergrid-test-context.xml
+++ b/stack/services/src/test/resources/usergrid-test-context.xml
@@ -51,7 +51,7 @@
<constructor-arg ref="cassandraCluster"/>
</bean>
- <bean id="binaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
+ <bean id="localFileBinaryStore" class="org.apache.usergrid.services.assets.data.LocalFileBinaryStore">
<property name="reposLocation" value="${usergrid.temp.files}"/>
</bean>