You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/10 21:26:09 UTC
[16/50] [abbrv] git commit: Removed admin token from export call. The
admin token must be supplied as a param instead. Removed extra code from
ExportServiceImpl. Now only prints out all the collections in an application.
Currently excludes only the expor
Removed admin token from export call. The admin token must be supplied as a param instead. Removed extra code from ExportServiceImpl. Now only prints out all the collections in an application. Currently excludes only the exports collections.
Removed old integrations tests, and replaced it with a working integration test.
Added tests that check to make sure it only limits the scope of the export to a specific app in a specific org that it was called from.
Fixed fileValidity/Connection due to error in for loop.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b0bc12d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b0bc12d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b0bc12d9
Branch: refs/pull/70/merge
Commit: b0bc12d9aec310d52f3d77f5c9a7b57a1d74f8bf
Parents: ae31d97
Author: grey <gr...@apigee.com>
Authored: Mon Feb 24 14:38:18 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Mon Feb 24 14:38:18 2014 -0800
----------------------------------------------------------------------
.../org/usergrid/management/ExportInfo.java | 6 +-
.../management/export/ExportServiceImpl.java | 70 +----
.../cassandra/ManagementServiceIT.java | 293 +++++++++----------
3 files changed, 144 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0bc12d9/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
index d0d2630..7821ca6 100644
--- a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
+++ b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
@@ -18,7 +18,6 @@ public class ExportInfo extends TypedEntity {
private String storage_provider;
private Map<String, Object> storage_info;
private String s3_accessId;
- private String admin_token;
private String s3_key;
private String bucket_location;
private UUID applicationId;
@@ -29,7 +28,6 @@ public class ExportInfo extends TypedEntity {
storage_provider = (String) properties.get ("storage_provider");
storage_info = (Map) properties.get("storage_info");
s3_accessId = (String) storage_info.get("s3_accessId");
- admin_token = (String) storage_info.get("admin_token");
s3_key = (String) storage_info.get("s3_key");
bucket_location = (String) storage_info.get("bucket_location");
}
@@ -42,8 +40,7 @@ public class ExportInfo extends TypedEntity {
public String getPath () {
return path;
- };
-
+ }
//Wouldn't get exposed.
public Map<String, Object> getProperties() {
@@ -57,7 +54,6 @@ public class ExportInfo extends TypedEntity {
public Map<String, Object> getStorage_info () { return storage_info; }
- public String getAdmin_token () { return admin_token; }
//TODO: is this a security concern? How would we get rid of the key once we're done with this value?
public String getS3_key () { return s3_key; }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0bc12d9/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
index 317dec3..498828b 100644
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
@@ -159,11 +159,6 @@ public class ExportServiceImpl implements ExportService {
em.update( export );
-// Map<UUID, String> organizationGet = getOrg(pathItems[0]);
-// if (organizationGet == null) {
-// logger.error( "couldn't parse your organization" );
-// return;
-// }
Map<UUID, String> organizationGet = getOrgs(config);
for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
//needs to pass app name, and possibly collection to export
@@ -173,24 +168,6 @@ public class ExportServiceImpl implements ExportService {
em.update( export );
}
- //now we also need somebody to take the export info and look through the path for the specific org.
- //That way we get a specific org instead of getting all the orgs.
-// private Map<UUID, String> getOrg(String orgName) throws Exception {
-// Map<UUID, String> organizationNames = null;
-// OrganizationInfo info = managementService.getOrganizationByName( orgName );
-// managementService.
-//
-// if ( info == null ) {
-// logger.error( "Organization info is null!" );
-// return null;
-// }
-//
-// organizationNames = new HashMap<UUID, String>();
-// organizationNames.put( info.getUuid(), info.getName() );
-//
-// return organizationNames;
-// }
-
private Map<UUID, String> getOrgs(ExportInfo exportInfo) throws Exception {
// Loop through the organizations
// TODO:this will come from the orgs in schedule when you do the validations. delete orgId
@@ -267,45 +244,9 @@ public class ExportServiceImpl implements ExportService {
JsonGenerator jg = getJsonGenerator( baos );
- // load the dictionary
- //TODO: change the CassService below to only be the applicationId that gets stored in the config.
- EntityManager rootEm = emf.getEntityManager( config.getApplicationId() );
-
- Entity appEntity = rootEm.get( application.getKey() );
-
- jobExecution.heartbeat();
- Map<String, Object> dictionaries = new HashMap<String, Object>();
- for ( String dictionary : rootEm.getDictionaries( appEntity ) ) {
- Map<Object, Object> dict = rootEm.getDictionaryAsMap( appEntity, dictionary );
-
- // nothing to do
- if ( dict.isEmpty() ) {
- continue;
- }
-
- dictionaries.put( dictionary, dict );
- }
- //TODO: resolve problem that look similar to this.
EntityManager em = emf.getEntityManager( application.getKey() );
- // Get application
- Entity nsEntity = em.get( application.getKey() );
-
- Set<String> collections = em.getApplicationCollections();
-
- // load app counters
-
- Map<String, Long> entityCounters = em.getApplicationCounters();
-
- nsEntity.setMetadata( "organization", organization );
- nsEntity.setMetadata( "dictionaries", dictionaries );
- // counters for collections
- nsEntity.setMetadata( "counters", entityCounters );
- nsEntity.setMetadata( "collections", collections );
-
- jobExecution.heartbeat();
jg.writeStartArray();
- jg.writeObject( nsEntity );
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
long starting_time = System.currentTimeMillis();
@@ -314,6 +255,9 @@ public class ExportServiceImpl implements ExportService {
// through the entities in the application (former namespace).
//could support queries, just need to implement that in the rest endpoint.
for ( String collectionName : metadata.keySet() ) {
+ if(collectionName.equals( "exports" )) {
+ continue;
+ }
Query query = new Query();
query.setLimit( MAX_ENTITY_FETCH );
@@ -340,7 +284,6 @@ public class ExportServiceImpl implements ExportService {
baos.flush();
baos.close();
-
InputStream is = new ByteArrayInputStream( baos.toByteArray() );
s3Export.copyToS3( is, config , appFileName);
}
@@ -492,14 +435,7 @@ public class ExportServiceImpl implements ExportService {
* @return the file name concatenated with the type and the name of the collection
*/
protected String prepareOutputFileName( String type, String name ) {
- //name = name.replace( "/", PATH_REPLACEMENT );
- // Add application and timestamp
StringBuilder str = new StringBuilder();
- // str.append(baseOutputFileName);
- // str.append(".");
- str.append( "collections" );
- //str.append( type );
- //str.append( "." );
str.append( name );
str.append( "." );
str.append( startTime );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b0bc12d9/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
index fcddd37..de62069 100644
--- a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
@@ -51,6 +51,7 @@ import org.usergrid.utils.UUIDUtils;
import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -742,44 +743,6 @@ public class ManagementServiceIT {
assertEquals( userId, authedUser.getUuid() );
}
- /*Make this test the do export test and verify that it works using a mock method. */
- //This test really should be called testDoExport as it mocks out sending it to s3.
- //the test below walks through very similar code as the following test.
-// @Test
-// public void testS3Export() throws Exception {
-//
-// HashMap<String, Object> payload = new HashMap<String, Object>();
-// Map<String, Object> properties = new HashMap<String, Object>();
-// Map<String, Object> storage_info = new HashMap<String, Object>();
-// storage_info.put( "admin_token","insert_token_data_here" );
-// //TODO: always put dummy values here and ignore this test.
-// //TODO: add a ret for when s3 values are invalid.
-// storage_info.put( "s3_key","insert key here" );
-// storage_info.put( "s3_accessId","insert access id here");
-// storage_info.put( "bucket_location","insert bucket name here");
-//
-//
-// properties.put( "storage_provider","s3");
-// properties.put( "storage_info",storage_info);
-//
-// payload.put( "path", "test-organization/test-app/user");
-// payload.put( "properties", properties);
-//
-//
-// ExportInfo exportInfo = new ExportInfo(payload);
-//
-//
-// S3Export s3Export = mock( S3Export.class );
-//
-// try {
-// setup.getExportService().setS3Export( s3Export );
-// setup.getExportService().doExport( exportInfo, );
-// }catch (Exception e) {
-// assert(false);
-// }
-// assert(true);
-// }
-
//Tests to make sure we can call the job with mock data and it runs.
@Test
public void testFileConnections() throws Exception {
@@ -836,22 +799,9 @@ public class ManagementServiceIT {
JSONParser parser = new JSONParser();
org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ assertEquals(13, a.size() );
- if (a.size() > 0) {
- org.json.simple.JSONObject objOrg = ( org.json.simple.JSONObject) a.get( 0 );
- String appName = (String) objOrg.get( "applicationName" );
-
- assertEquals("ed-application", appName );
-
- String path = (String) objOrg.get( "name" );
- assertEquals("ed-organization/ed-application",path );
-
- }
- else {
- assert(false);
- }
-
- org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject) a.get( 1 );
+ org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject) a.get( 0 );
org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject) objEnt.get( "connections" );
assertNotNull( objConnections );
@@ -865,14 +815,13 @@ public class ManagementServiceIT {
@Test
- public void testFileValidity() throws Exception {
+ public void testFileValidity() throws Exception {
File f = null;
-
try {
- f = new File ("test.json");
- f.delete();
+ f = new File ("test.json");
+ f.delete();
} catch (Exception e) {
//consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
}
@@ -901,27 +850,126 @@ public class ManagementServiceIT {
org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
- if (a.size() > 0) {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( 0 );
- String appName = (String) entity.get( "applicationName" );
+ for (int i = 0; i < a.size();i++ )
+ {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
+ assertNotNull( entityData );
+
+ }
+ f.delete();
+ }
- assertEquals("ed-application", appName );
+ @Test
+ public void testFileExportOneOrg() throws Exception {
- String path = (String) entity.get( "name" );
- assertEquals("ed-organization/ed-application",path );
+ File f = null;
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
}
- else {
- assert(false);
- }
+ setup.getMgmtSvc().createOwnerAndOrganization( "noExport","junkUserName","junkRealName","ugExport@usergrid.com","123456789" );
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
- for (int i = 1; a.size() < i;i++ )
+ assertEquals( 3 , a.size() );
+ for (int i = 0; i < a.size();i++ )
{
org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
- assertNotNull( entityData );
+ String entityName = ( String) entityData.get( "name" );
+ assertNotEquals( "junkRealName",entityName );
+
+ }
+ f.delete();
+ }
+ @Test
+ public void testFileExportOneApp() throws Exception {
+
+ File f = null;
+ String orgName = "ed-organization";
+ String appName = "testAppNotExported";
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
+ }
+
+ UUID appId = setup.getEmf().createApplication( orgName, appName );
+
+
+ EntityManager em = setup.getEmf().getEntityManager( appId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[10];
+ //creates entities
+ for (int i = 0; i< 10;i++) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ entity[i] = em.create( "user", userProperties );
+ }
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+
+ assertEquals( 3 , a.size() );
+ for (int i = 0; i < a.size();i++ )
+ {
+ org.json.simple.JSONObject data = ( org.json.simple.JSONObject) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
+ String entityName = ( String) entityData.get( "name" );
+ assertNotEquals( "junkRealName",entityName );
}
f.delete();
}
@@ -1038,119 +1086,58 @@ public class ManagementServiceIT {
assert(true);
}
- @Ignore
- public void testS3IntegrationExport100Entities() throws Exception {
-
- //EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- HashMap<String, Object> payload = new HashMap<String, Object>();
- Map<String, Object> properties = new HashMap<String, Object>();
- Map<String, Object> storage_info = new HashMap<String, Object>();
- storage_info.put( "admin_token","insert_token_data_here" );
- //TODO: always put dummy values here and ignore this test.
-
-
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
-
- payload.put( "path", "test-organization/test-app/user");
- payload.put( "properties", properties);
+ @Ignore //For this test please input your s3 credentials into payload builder.
+ public void testIntegration100Entities() throws Exception {
- ExportInfo exportInfo = new ExportInfo(payload);
S3Export s3Export = new S3ExportImpl();
- ExportJob job = new ExportJob();
- JobExecution jobExecution = mock (JobExecution.class);
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
- UUID uuid = UUIDUtils.newTimeUUID();
EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ //intialize user object to be posted
Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[100];
+ //creates entities
for (int i = 0; i< 100;i++) {
- userProperties = new LinkedHashMap<String, Object>();
+ userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
- em.create( "user", userProperties );
+ entity[i] = em.create( "user", userProperties );
}
- try {
- setup.getExportService().setS3Export( s3Export );
- setup.getExportService().doExport( exportInfo, jobExecution );
- }catch (Exception e) {
- e.printStackTrace();
- assert(false);
- }
- assert(true);
- }
-
- @Ignore
- public void testS3IntegrationExport10Connections() throws Exception {
-
- //EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- HashMap<String, Object> payload = new HashMap<String, Object>();
- Map<String, Object> properties = new HashMap<String, Object>();
- Map<String, Object> storage_info = new HashMap<String, Object>();
- storage_info.put( "admin_token","insert_token_data_here" );
- //TODO: always put dummy values here and ignore this test.
-
-
-
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
-
- payload.put( "path", "test-organization/test-app/user");
- payload.put( "properties", properties);
-
-
- ExportInfo exportInfo = new ExportInfo(payload);
-
-
- S3Export s3Export = new S3ExportImpl();
- JobExecution jobExecution = mock (JobExecution.class);
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+ //create and initialize jobData returned in JobExecution.
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
- UUID uuid = UUIDUtils.newTimeUUID();
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
- Map<String, Object> userProperties = null;
- Entity[] entity;
- entity = new Entity[10];
- for (int i = 0; i< 10;i++) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
- entity[i] = em.create( "user", userProperties );
+ exportService.doExport( exportInfo, jobExecution );
- }
- //em.createConnection( )
- em.createConnection( em.getRef( entity[0].getUuid() ),"Likes",em.getRef( entity[1].getUuid() ) );
- em.createConnection( em.getRef( entity[1].getUuid() ),"Likes",em.getRef( entity[0].getUuid() ) );
- try {
- setup.getExportService().setS3Export( s3Export );
- setup.getExportService().doExport( exportInfo, jobExecution );
- }catch (Exception e) {
- e.printStackTrace();
- assert(false);
- }
- assert(true);
}
+
/*Creates fake payload for testing purposes.*/
public HashMap<String,Object> payloadBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> properties = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
- //TODO: make sure to put a valid admin token here.
- //storage_info.put( "admin_token","insert_token_data_here" );
- //TODO: always put dummy values here and ignore this test.
- //TODO: add a ret for when s3 values are invalid.
+// TODO: always put dummy values here and ignore this test.
storage_info.put( "s3_key","insert key here" );
storage_info.put( "s3_accessId","insert access id here");
storage_info.put( "bucket_location","insert bucket name here");
-
properties.put( "storage_provider","s3");
properties.put( "storage_info",storage_info);