You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/03/30 21:58:12 UTC
[08/50] incubator-usergrid git commit: Fixes to REST module import
tests.
Fixes to REST module import tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b7e41f25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b7e41f25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b7e41f25
Branch: refs/heads/two-dot-o-dev
Commit: b7e41f2570540a006291418f9870b008290cee4c
Parents: 10481ce
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Mar 6 12:30:12 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Mar 6 12:30:12 2015 -0500
----------------------------------------------------------------------
.../applications/imports/ImportsResource.java | 73 +++++++----------
.../rest/management/ImportResourceIT.java | 82 +++++++++++---------
2 files changed, 75 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b7e41f25/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
index aa5160b..859f0f5 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
@@ -37,6 +37,8 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.PathSegment;
import javax.ws.rs.core.UriInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
@@ -63,6 +65,7 @@ import com.sun.jersey.api.json.JSONWithPadding;
@Produces( MediaType.APPLICATION_JSON )
public class ImportsResource extends AbstractContextResource {
+ private static final Logger logger = LoggerFactory.getLogger(ImportsResource.class);
@Autowired
protected ImportService importService;
@@ -94,10 +97,7 @@ public class ImportsResource extends AbstractContextResource {
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
-
ApiResponse response = createApiResponse();
-
-
response.setAction( "post" );
response.setApplication( emf.getEntityManager( application.getId() ).getApplication() );
response.setParams( ui.getQueryParameters() );
@@ -106,42 +106,33 @@ public class ImportsResource extends AbstractContextResource {
Map<String, Object> properties;
Map<String, Object> storage_info;
- // UsergridAwsCredentialsProvider uacp = new UsergridAwsCredentialsProvider();
-
- // try {
- //checkJsonExportProperties(json);
-
-
- if ( ( properties = ( Map<String, Object> ) json.get( "properties" ) ) == null ) {
- throw new NullArgumentException( "Could not find 'properties'" );
- }
- storage_info = ( Map<String, Object> ) properties.get( "storage_info" );
- String storage_provider = ( String ) properties.get( "storage_provider" );
- if ( storage_provider == null ) {
- throw new NullArgumentException( "Could not find field 'storage_provider'" );
- }
- if ( storage_info == null ) {
- throw new NullArgumentException( "Could not find field 'storage_info'" );
- }
-
- String bucketName = ( String ) storage_info.get( "bucket_location" );
+ if ((properties = (Map<String, Object>) json.get("properties")) == null) {
+ throw new NullArgumentException("Could not find 'properties'");
+ }
+ storage_info = (Map<String, Object>) properties.get("storage_info");
+ String storage_provider = (String) properties.get("storage_provider");
+ if (storage_provider == null) {
+ throw new NullArgumentException("Could not find field 'storage_provider'");
+ }
+ if (storage_info == null) {
+ throw new NullArgumentException("Could not find field 'storage_info'");
+ }
- String accessId = ( String ) storage_info.get( "s3_access_id" );
- String secretKey = ( String ) storage_info.get( "s3_key" );
-
- if ( bucketName == null ) {
- throw new NullArgumentException( "Could not find field 'bucketName'" );
- }
- if ( accessId == null ) {
- throw new NullArgumentException( "Could not find field 's3_access_id'" );
- }
- if ( secretKey == null ) {
-
- throw new NullArgumentException( "Could not find field 's3_key'" );
- }
+ String bucketName = (String) storage_info.get("bucket_location");
+ String accessId = (String) storage_info.get("s3_access_id");
+ String secretKey = (String) storage_info.get("s3_key");
+ if (bucketName == null) {
+ throw new NullArgumentException("Could not find field 'bucketName'");
+ }
+ if (accessId == null) {
+ throw new NullArgumentException("Could not find field 's3_access_id'");
+ }
+ if (secretKey == null) {
+ throw new NullArgumentException("Could not find field 's3_key'");
+ }
json.put( "organizationId", organization.getUuid() );
json.put( "applicationId", application.getId() );
@@ -157,7 +148,6 @@ public class ImportsResource extends AbstractContextResource {
@GET
public JSONWithPadding getImports( @Context UriInfo ui, @QueryParam( "ql" ) String query, @QueryParam( "cursor" ) String cursor ) throws Exception {
-
final Results importResults = importService.getImports( application.getId(), query, cursor );
if ( importResults == null ) {
@@ -165,13 +155,9 @@ public class ImportsResource extends AbstractContextResource {
}
ApiResponse response = createApiResponse();
-
-
response.setAction( "get" );
response.setApplication( emf.getEntityManager( application.getId() ).getApplication() );
response.setParams( ui.getQueryParameters() );
-
-
response.withResults( importResults );
return new JSONWithPadding( response );
@@ -186,18 +172,17 @@ public class ImportsResource extends AbstractContextResource {
final UUID importId = UUID.fromString( entityId.getPath() );
final Import importEntity = importService.getImport( application.getId(), importId );
+ logger.debug("Loaded import entity {}:{} with state {}",
+ new Object[] { importEntity.getType(), importEntity.getUuid(), importEntity.getState() } );
+
if ( importEntity == null ) {
throw new EntityNotFoundException( "could not find import with uuid " + importId );
}
ApiResponse response = createApiResponse();
-
-
response.setAction( "get" );
response.setApplication( emf.getEntityManager( application.getId() ).getApplication() );
response.setParams( ui.getQueryParameters() );
-
-
response.setEntities( Collections.<Entity>singletonList( importEntity ) );
return new JSONWithPadding( response );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b7e41f25/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
index 2edeb65..1ea9770 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ImportResourceIT.java
@@ -29,7 +29,6 @@ import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.batch.service.JobSchedulerService;
import org.apache.usergrid.management.importer.S3Upload;
-import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.test.resource2point0.AbstractRestIT;
import org.apache.usergrid.rest.test.resource2point0.model.ApiResponse;
@@ -88,7 +87,9 @@ public class ImportResourceIT extends AbstractRestIT {
bucketPrefix = System.getProperty("bucketName");
// start the scheduler after we're all set up
- JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean( JobSchedulerService.class );
+ JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance()
+ .getSpringResource().getBean( JobSchedulerService.class );
+
if (jobScheduler.state() != Service.State.RUNNING) {
jobScheduler.startAsync();
jobScheduler.awaitRunning();
@@ -113,9 +114,9 @@ public class ImportResourceIT extends AbstractRestIT {
});
}
- if (!StringUtils.isEmpty(bucketPrefix)) {
- deleteBucketsWithPrefix();
- }
+// if (!StringUtils.isEmpty(bucketPrefix)) {
+// deleteBucketsWithPrefix();
+// }
bucketName = bucketPrefix + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
}
@@ -172,15 +173,11 @@ public class ImportResourceIT extends AbstractRestIT {
Entity payload = payloadBuilder();
// /management/orgs/orgname/apps/appname/import
- Entity entity = this.management()
- .orgs()
- .organization(org)
- .app()
+ Entity entity = this.management().orgs().organization(org).app()
.addToPath(app)
.addToPath("imports")
.post(payload);
-
assertNotNull(entity);
// test that you can access the organization using the currently set token.
@@ -421,7 +418,7 @@ public class ImportResourceIT extends AbstractRestIT {
+ File.separator + "test-classes" + File.separator;
List<String> filenames = new ArrayList<>( 1 );
- filenames.add( basePath + "testImportCorrect.testCol.1.json" );
+ filenames.add( basePath + "testimport-correct-testcol.1.json");
// create 10 applications each with collection of 10 things, export all to S3
S3Upload s3Upload = new S3Upload();
@@ -458,7 +455,7 @@ public class ImportResourceIT extends AbstractRestIT {
final Entity includesEntity = importGetIncludesResponse.getEntities().get( 0 );
- assertTrue( includesEntity.getAsString("fileName").endsWith("testImportCorrect.testCol.1.json"));
+ assertTrue( includesEntity.getAsString("fileName").endsWith("testimport-correct-testcol.1.json"));
assertEquals(1, includesEntity.get( "importedConnectionCount" ));
assertEquals(1, includesEntity.get( "importedEntityCount" ));
@@ -493,7 +490,7 @@ public class ImportResourceIT extends AbstractRestIT {
+ File.separator + "test-classes" + File.separator;
List<String> filenames = new ArrayList<>( 2 );
- filenames.add( basePath + "testImportCorrect.testCol.1.json");
+ filenames.add( basePath + "testimport-correct-testcol.1.json");
filenames.add( basePath + "testImport.testApplication.2.json");
// create 10 applications each with collection of 10 things, export all to S3
@@ -522,8 +519,6 @@ public class ImportResourceIT extends AbstractRestIT {
assertNotNull(collection);
assertEquals(1, collection.getNumOfEntities());
assertEquals("thing0", collection.getResponse().getEntities().get(0).get("name"));
-
-
}
/**
@@ -554,16 +549,17 @@ public class ImportResourceIT extends AbstractRestIT {
Entity importEntity = importCollection();
- Entity importGet = this.management().orgs().organization(org).app().addToPath(app)
- .addToPath("imports" ).addToPath(importEntity.getUuid().toString() ).get();
-
+ Entity importGet = this.management().orgs().organization(org).app()
+ .addToPath(app)
+ .addToPath("imports" )
+ .addToPath(importEntity.getUuid().toString() )
+ .get();
assertNotNull(importGet);
assertEquals("FAILED", importGet.get("state"));
assertEquals(1, importGet.get("fileCount"));
-
Collection collection = this.app().collection("things").get();
assertNotNull(collection);
@@ -571,14 +567,14 @@ public class ImportResourceIT extends AbstractRestIT {
}
-//export with two files and import the two files.
- //also test the includes endpoint.
+
/**
* TODO: Test that importing bad JSON will result in an informative error message.
*/
@Test
public void testImportBadJson() throws Exception {
+
// import from a bad JSON file
Assume.assumeTrue(configured);
@@ -589,7 +585,7 @@ public class ImportResourceIT extends AbstractRestIT {
+ File.separator + "test-classes" + File.separator;
List<String> filenames = new ArrayList<>( 1 );
- filenames.add( basePath + "testImportInvalidJson.testApplication.3.json");
+ filenames.add( basePath + "testimport-bad-json-testapp.3.json");
// create 10 applications each with collection of 10 things, export all to S3
S3Upload s3Upload = new S3Upload();
@@ -604,14 +600,21 @@ public class ImportResourceIT extends AbstractRestIT {
// we should now have 100 Entities in the default app
- Entity importGet = this.management().orgs().organization( org ).app().addToPath( app ).addToPath("imports")
- .addToPath( importEntity.getUuid().toString() ).get();
+ Entity importGet = this.management().orgs().organization( org ).app()
+ .addToPath( app )
+ .addToPath("imports")
+ .addToPath( importEntity.getUuid().toString() )
+ .get();
- Entity importGetIncludes = this.management().orgs().organization(org).app().addToPath(app)
- .addToPath("imports" ).addToPath(importEntity.getUuid().toString() )
- .addToPath("files" ).get();
+ Entity importGetIncludes = this.management().orgs().organization(org).app()
+ .addToPath(app)
+ .addToPath("imports" )
+ .addToPath(importEntity.getUuid().toString() )
+ .addToPath("files" )
+ .get();
assertNotNull(importGet);
+
//TODO: needs better error checking
assertNotNull(importGetIncludes);
@@ -641,8 +644,10 @@ public class ImportResourceIT extends AbstractRestIT {
}});
}});
- Entity importEntity = this.management().orgs().organization(org).app().addToPath(app).addToPath("imports")
- .post(importPayload);
+ Entity importEntity = this.management().orgs().organization(org).app()
+ .addToPath(app)
+ .addToPath("imports")
+ .post(importPayload);
int maxRetries = 120;
int retries = 0;
@@ -766,17 +771,22 @@ public class ImportResourceIT extends AbstractRestIT {
/*Creates fake payload for testing purposes.*/
public Entity payloadBuilder() {
- Entity payload = new Entity();
- Entity properties = new Entity();
+
+ String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
+ String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
+
Entity storage_info = new Entity();
- //TODO: always put dummy values here and ignore this test.
- //TODO: add a ret for when s3 values are invalid.
- storage_info.put("s3_key", "insert key here");
- storage_info.put("s3_access_id", "insert access id here");
- storage_info.put("bucket_location", "insert bucket name here");
+ storage_info.put("s3_key", secretKey);
+ storage_info.put("s3_access_id", accessId);
+ storage_info.put("bucket_location", bucketName) ;
+
+ Entity properties = new Entity();
properties.put("storage_provider", "s3");
properties.put("storage_info", storage_info);
+
+ Entity payload = new Entity();
payload.put("properties", properties);
+
return payload;
}
}