You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/04/01 03:20:58 UTC
[06/27] git commit: Add query tests, but doesn’t protect the users from themselves. Reduced number of max entities so people don’t crash the JVM with huge queries.
Add query tests, but doesn’t protect the users from themselves.
Reduced number of max entities so people don’t crash the JVM with huge queries.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1a5f8b47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1a5f8b47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1a5f8b47
Branch: refs/pull/92/merge
Commit: 1a5f8b479e4dfbfc4036032d6814b931fa9e7aa0
Parents: 844fd5d
Author: grey <gr...@apigee.com>
Authored: Mon Mar 31 11:28:11 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Mon Mar 31 11:28:11 2014 -0700
----------------------------------------------------------------------
.../management/export/ExportServiceImpl.java | 30 ++++-
.../cassandra/ManagementServiceIT.java | 109 +++++++++++++++----
2 files changed, 113 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1a5f8b47/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 053a4e3..92e76bc 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -71,7 +71,7 @@ public class ExportServiceImpl implements ExportService {
private ManagementService managementService;
//Maximum amount of entities retrieved in a single go.
- public static final int MAX_ENTITY_FETCH = 100000;
+ public static final int MAX_ENTITY_FETCH = 1000;
//Amount of time that has passed before sending another heart beat in millis
public static final int TIMESTAMP_DELTA = 5000;
@@ -314,7 +314,13 @@ public class ExportServiceImpl implements ExportService {
if ( ( config.get( "collectionName" ) == null ) || collectionName
.equals( config.get( "collectionName" ) ) ) {
//Query entity manager for the entities in a collection
- Query query = new Query();
+ Query query;
+ if(config.get( "query" ) == null) {
+ query = new Query( );
+ }
+ else {
+ query = Query.fromQL( ( String ) config.get( "query" ) );
+ }
query.setLimit( MAX_ENTITY_FETCH );
query.setResultsLevel( Results.Level.ALL_PROPERTIES );
Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
@@ -394,9 +400,19 @@ public class ExportServiceImpl implements ExportService {
if ( ( config.get( "collectionName" ) == null ) || collectionName
.equals( config.get( "collectionName" ) ) ) {
//Query entity manager for the entities in a collection
- Query query = new Query();
+ Query query;
+ if(config.get( "query" ) == null) {
+ query = new Query( );
+ }
+ else {
+ query = Query.fromQL( ( String ) config.get( "query" ) );
+ }
+ // Query query = Query.fromQL( ( String ) config.get( "query" ) ); //new Query();
query.setLimit( MAX_ENTITY_FETCH );
query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+ query.setCollection( collectionName );
+ //query.setQl( ( String ) config.get( "query" ) );
+
Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
//pages through the query and backs up all results.
@@ -467,7 +483,13 @@ public class ExportServiceImpl implements ExportService {
//if the collection you are looping through doesn't match the name of the one you want. Don't export it.
if ( collectionName.equals( ( String ) config.get( "collectionName" ) ) ) {
//Query entity manager for the entities in a collection
- Query query = new Query();
+ Query query;
+ if(config.get( "query" ) == null) {
+ query = new Query( );
+ }
+ else {
+ query = Query.fromQL( ( String ) config.get( "query" ) );
+ }
query.setLimit( MAX_ENTITY_FETCH );
query.setResultsLevel( Results.Level.ALL_PROPERTIES );
Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1a5f8b47/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index fa8dd0c..cb817f0 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -1112,6 +1112,8 @@ public class ManagementServiceIT {
catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
}
+ f.deleteOnExit();
+
UUID appId = setup.getEmf().createApplication( orgName, appName );
@@ -1161,27 +1163,25 @@ public class ManagementServiceIT {
String entityName = ( String ) entityData.get( "name" );
assertFalse( "junkRealName".equals( entityName ) );
}
- f.deleteOnExit();
}
-//
+
@Test
- public void testExportOneAppOnApplicationEndpoint() throws Exception {
+ public void testExportOneAppOnApplicationEndpointWQuery() throws Exception {
File f = null;
- String orgName = "ed-organization";
- String appName = "testAppNotExported";
try {
- f = new File( "exportOneApp.json" );
+ f = new File( "exportOneAppWQuery.json" );
}
catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
}
- UUID appId = setup.getEmf().createApplication( orgName, appName );
+ f.deleteOnExit();
- EntityManager em = setup.getEmf().getEntityManager( appId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
//intialize user object to be posted
Map<String, Object> userProperties = null;
Entity[] entity;
@@ -1189,16 +1189,19 @@ public class ManagementServiceIT {
//creates entities
for ( int i = 0; i < 1; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put("name","me");
userProperties.put( "username", "junkRealName");
- userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
+ userProperties.put( "email", "burp" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
entity[i] = em.create( "users", userProperties );
}
S3Export s3Export = new MockS3ExportImpl();
- s3Export.setFilename( "exportOneApp.json" );
+ s3Export.setFilename( "exportOneAppWQuery.json" );
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
+ payload.put( "query","select * where username = 'junkRealName'" );
+
payload.put( "organizationId",organization.getUuid() );
payload.put( "applicationId",applicationId);
@@ -1219,17 +1222,15 @@ public class ManagementServiceIT {
org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- assertEquals( 3 , a.size() );
+ assertEquals( 1 , a.size() );
for ( int i = 0; i < a.size(); i++ ) {
org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
String entityName = ( String ) entityData.get( "name" );
assertFalse( "junkRealName".equals( entityName ) );
}
-
- f.deleteOnExit();
-
}
+
//
@Test
public void testExportOneCollection() throws Exception {
@@ -1245,7 +1246,7 @@ public class ManagementServiceIT {
}
EntityManager em = setup.getEmf().getEntityManager( applicationId);
- em.createApplicationCollection( "baconators" );
+ em.createApplicationCollection( "qt" );
//intialize user object to be posted
Map<String, Object> userProperties = null;
Entity[] entity;
@@ -1255,7 +1256,7 @@ public class ManagementServiceIT {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
- entity[i] = em.create( "baconators", userProperties );
+ entity[i] = em.create( "qts", userProperties );
}
S3Export s3Export = new MockS3ExportImpl();
@@ -1265,7 +1266,7 @@ public class ManagementServiceIT {
payload.put( "organizationId",organization.getUuid() );
payload.put( "applicationId",applicationId);
- payload.put( "collectionName","baconators");
+ payload.put( "collectionName","qts");
UUID exportUUID = exportService.schedule( payload );
exportService.setS3Export( s3Export );
@@ -1289,6 +1290,70 @@ public class ManagementServiceIT {
}
+ @Test
+ public void testExportOneCollectionWQuery() throws Exception {
+
+ File f = null;
+ int entitiesToCreate = 5;
+
+ try {
+ f = new File( "exportOneCollectionWQuery.json" );
+ }
+ catch ( Exception e ) {
+ //consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
+ }
+ f.deleteOnExit();
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId);
+ em.createApplicationCollection( "baconators" );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[entitiesToCreate];
+ //creates entities
+ for ( int i = 0; i < entitiesToCreate; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
+ entity[i] = em.create( "baconators", userProperties );
+ }
+
+ S3Export s3Export = new MockS3ExportImpl();
+ s3Export.setFilename( "exportOneCollectionWQuery.json" );
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+ payload.put( "query","select * where username contains 'billybob0'" );
+
+ payload.put( "organizationId",organization.getUuid() );
+ payload.put( "applicationId",applicationId);
+ payload.put( "collectionName","baconators");
+
+ UUID exportUUID = exportService.schedule( payload );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", payload );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ exportService.doExport( jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+
+ //only one entity should match the query.
+ assertEquals( 1 , a.size() );
+
+ }
+
+
+
+
+
//@Ignore("file created won't be deleted when running tests")
@Test
public void testExportOneOrganization() throws Exception {
@@ -1503,11 +1568,11 @@ public class ManagementServiceIT {
//intialize user object to be posted
Map<String, Object> userProperties = null;
Entity[] entity;
- entity = new Entity[100];
+ entity = new Entity[5];
//creates entities
ApplicationInfo appMade = null;
- for(int i = 0; i < 100; i++) {
+ for(int i = 0; i < 5; i++) {
appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp"+i);
EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
@@ -1515,9 +1580,9 @@ public class ManagementServiceIT {
//intialize user object to be posted
Map<String, Object> entityLevelProperties = null;
Entity[] entNotCopied;
- entNotCopied = new Entity[20];
+ entNotCopied = new Entity[5];
//creates entities
- for ( int index = 0; index < 20; index++ ) {
+ for ( int index = 0; index < 5; index++ ) {
entityLevelProperties = new LinkedHashMap<String, Object>();
entityLevelProperties.put( "username", "bobso" + index );
entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
@@ -1710,7 +1775,7 @@ public class ManagementServiceIT {
OrganizationInfo orgMade = null;
ApplicationInfo appMade = null;
for(int i = 0; i < 100; i++) {
- orgMade =setup.getMgmtSvc().createOrganization( "minorboss"+i,adminUser,true );
+ orgMade =setup.getMgmtSvc().createOrganization( "largerboss"+i,adminUser,true );
appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp"+i);
EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );