You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/07/22 17:17:54 UTC
[13/15] incubator-usergrid git commit: Added data creator to generate
data for ExportApp testing.
Added data creator to generate data for ExportApp testing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a6e68a0a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a6e68a0a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a6e68a0a
Branch: refs/heads/two-dot-o
Commit: a6e68a0adc571b0a30138d4cb71b5e0e49a42c0a
Parents: 2748347
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Jul 21 14:59:59 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Jul 21 14:59:59 2015 -0400
----------------------------------------------------------------------
stack/pom.xml | 2 +-
stack/tools/pom.xml | 6 +
.../org/apache/usergrid/tools/ExportApp.java | 17 +-
.../usergrid/tools/ExportDataCreator.java | 244 +++++++++++++++----
.../org/apache/usergrid/tools/ToolBase.java | 2 +-
.../apache/usergrid/tools/ExportAppTest.java | 65 +----
6 files changed, 223 insertions(+), 113 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a6e68a0a/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index da1b62c..0e1b32d 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -103,7 +103,7 @@
<org.springframework.version>3.1.2.RELEASE</org.springframework.version>
<shiro-version>1.2.0</shiro-version>
<slf4j-version>1.6.1</slf4j-version>
- <snakeyaml-version>1.8</snakeyaml-version>
+ <snakeyaml-version>1.9</snakeyaml-version>
<tomcat-version>7.0.42</tomcat-version>
<antlr.version>3.4</antlr.version>
<tika.version>1.4</tika.version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a6e68a0a/stack/tools/pom.xml
----------------------------------------------------------------------
diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml
index 4be3232..3402612 100644
--- a/stack/tools/pom.xml
+++ b/stack/tools/pom.xml
@@ -249,5 +249,11 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.codearte.jfairy</groupId>
+ <artifactId>jfairy</artifactId>
+ <version>0.4.3</version>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a6e68a0a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
index b2da0ea..db975e6 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
@@ -102,7 +102,7 @@ public class ExportApp extends ExportingToolBase {
*/
@Override
public void runTool(CommandLine line) throws Exception {
-
+
applicationName = line.getOptionValue( APPLICATION_NAME );
if (StringUtils.isNotEmpty( line.getOptionValue( WRITE_THREAD_COUNT ) )) {
@@ -122,8 +122,11 @@ public class ExportApp extends ExportingToolBase {
logger.info( "Export directory: " + outputDir.getAbsolutePath() );
startSpring();
-
+
UUID applicationId = emf.lookupApplication( applicationName );
+ if (applicationId == null) {
+ throw new RuntimeException( "Cannot find application " + applicationName );
+ }
final EntityManager em = emf.getEntityManager( applicationId );
organizationName = em.getApplication().getOrganizationName();
@@ -133,8 +136,9 @@ public class ExportApp extends ExportingToolBase {
Observable<String> collectionsObservable = Observable.create( new CollectionsObservable( em ) );
collectionsObservable.flatMap( new Func1<String, Observable<ExportEntity>>() {
-
+
public Observable<ExportEntity> call(String collection) {
+
return Observable.create( new EntityObservable( em, collection ) )
.doOnNext( new EntityWriteAction() ).subscribeOn( writeScheduler );
}
@@ -142,16 +146,17 @@ public class ExportApp extends ExportingToolBase {
}, writeThreadCount ).flatMap( new Func1<ExportEntity, Observable<ExportConnection>>() {
public Observable<ExportConnection> call(ExportEntity exportEntity) {
+
return Observable.create( new ConnectionsObservable( em, exportEntity ) )
.doOnNext( new ConnectionWriteAction() ).subscribeOn( writeScheduler );
}
- }, writeThreadCount)
+ }, writeThreadCount )
.doOnCompleted( new FileWrapUpAction() )
.toBlocking().last();
}
-
-
+
+
// ----------------------------------------------------------------------------------------
// reading data
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a6e68a0a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java
index a3d6517..6fa4896 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java
@@ -17,90 +17,232 @@
package org.apache.usergrid.tools;
-import java.util.UUID;
-
-import org.apache.usergrid.management.ManagementService;
+import io.codearte.jfairy.Fairy;
+import io.codearte.jfairy.producer.company.Company;
+import io.codearte.jfairy.producer.person.Person;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.OrganizationOwnerInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.entities.Activity;
-import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
-import static org.junit.Assert.assertNotNull;
+import java.util.*;
/**
- * Simple class to create test for for exporting
- *
- * @author tnine
+ * Create an app full of users and data.
*/
-public class ExportDataCreator {
+public class ExportDataCreator extends ToolBase {
+
+ public static final String APP_NAME = "application";
+ public static final String ORG_NAME = "organization";
+ public static final String NUM_USERS = "users";
+ public static final String NUM_COLLECTIONS = "collections";
+ public static final String NUM_ENTITIES = "entities";
+ public static final String ADMIN_USERNAME = "username";
+ public static final String ADMIN_PASSWORD = "password";
- private EntityManagerFactory emf;
+ public String appName = "test-app";
+ public String orgName = "test-organization";
+ public int numUsers = 100;
+ public int numCollections = 20;
+ public int numEntities = 100;
+ public String adminUsername = "adminuser";
+ public String adminPassword = "test";
- private ManagementService managementService;
+ @Override
+ public void runTool(CommandLine line) throws Exception {
- /**
- * @param emf
- * @param managementService
- */
- public ExportDataCreator( EntityManagerFactory emf, ManagementService managementService ) {
- super();
- this.emf = emf;
- this.managementService = managementService;
+ startSpring();
+
+ setVerbose( line );
+
+ if (line.hasOption( APP_NAME )) {
+ appName = line.getOptionValue( APP_NAME );
+ }
+ if (line.hasOption( ORG_NAME )) {
+ orgName = line.getOptionValue( ORG_NAME );
+ }
+ if (line.hasOption( NUM_USERS )) {
+ numUsers = Integer.parseInt( line.getOptionValue( NUM_USERS ) );
+ }
+ if (line.hasOption( NUM_COLLECTIONS )) {
+ numCollections = Integer.parseInt( line.getOptionValue( NUM_COLLECTIONS ) );
+ }
+ if (line.hasOption( NUM_ENTITIES )) {
+ numEntities = Integer.parseInt( line.getOptionValue( NUM_ENTITIES ) );
+ }
+ if (line.hasOption( ADMIN_USERNAME )) {
+ adminUsername = line.getOptionValue( ADMIN_USERNAME );
+ }
+ if (line.hasOption( ADMIN_PASSWORD )) {
+ adminPassword = line.getOptionValue( ADMIN_PASSWORD );
+ }
+
+ createTestData();
}
- public void createTestData() throws Exception {
+ @Override
+ @SuppressWarnings("static-access")
+ public Options createOptions() {
- String orgName = "testexportorg";
+ Options options = super.createOptions();
- //nothing to do
- if ( managementService.getOrganizationByName( orgName ) != null ) {
- return;
- }
+ Option appName = OptionBuilder.hasArg()
+ .withDescription( "Application name to use" ).create( APP_NAME );
- OrganizationOwnerInfo orgInfo = managementService
- .createOwnerAndOrganization( orgName, "textExportUser@apigee.com", "Test User",
- "textExportUser@apigee.com", "password", true, false );
+ Option orgName = OptionBuilder.hasArg()
+ .withDescription( "Organization to use (will create if not present)" ).create( ORG_NAME );
- UUID appId = managementService.createApplication( orgInfo.getOrganization().getUuid(), "application" ).getId();
+ Option numUsers = OptionBuilder.hasArg()
+ .withDescription( "Number of users create (in addition to users)" ).create( NUM_USERS );
- EntityManager em = emf.getEntityManager( appId );
+ Option numCollection = OptionBuilder.hasArg()
+ .withDescription( "Number of collections to create (in addition to users)" ).create( NUM_COLLECTIONS );
- User first = new User();
- first.setUsername( "first" );
- first.setEmail( "first@usergrid.com" );
+ Option numEntities = OptionBuilder.hasArg()
+ .withDescription( "Number of entities to create per collection" ).create( NUM_ENTITIES );
- Entity firstUserEntity = em.create( first );
+ Option adminUsername = OptionBuilder.hasArg()
+ .withDescription( "Admin Username" ).create( ADMIN_USERNAME );
- assertNotNull( firstUserEntity );
+ Option adminPassword = OptionBuilder.hasArg()
+ .withDescription( "Admin Password" ).create( ADMIN_PASSWORD );
- User second = new User();
- second.setUsername( "second" );
- second.setEmail( "second@usergrid.com" );
+ options.addOption( appName );
+ options.addOption( orgName );
+ options.addOption( numUsers );
+ options.addOption( numCollection );
+ options.addOption( numEntities );
+ options.addOption( adminUsername );
+ options.addOption( adminPassword );
- Entity secondUserEntity = em.create( second );
+ return options;
+ }
- assertNotNull( secondUserEntity );
- em.createConnection( firstUserEntity, "likes", secondUserEntity );
+ public void createTestData() throws Exception {
- em.createConnection( secondUserEntity, "dislikes", firstUserEntity );
+ OrganizationInfo orgInfo = managementService.getOrganizationByName( orgName );
- // now create some activities and put them into the user stream
+ if (orgInfo == null) {
+ OrganizationOwnerInfo ownerInfo = managementService.createOwnerAndOrganization(
+ orgName, adminUsername + "@example.com", adminUsername,
+ adminUsername + "@example.com", adminPassword, true, false );
+ orgInfo = ownerInfo.getOrganization();
+ }
- Activity activity = new Activity();
+ ApplicationInfo appInfo = managementService.getApplicationInfo( orgName + "/" + appName );
- Activity.ActivityObject actor = new Activity.ActivityObject();
- actor.setEntityType( "user" );
- actor.setId( firstUserEntity.getUuid().toString() );
+ if (appInfo == null) {
+ UUID appId = managementService.createApplication( orgInfo.getUuid(), appName ).getId();
+ appInfo = managementService.getApplicationInfo( appId );
+ }
- activity.setActor( actor );
- activity.setVerb( "POST" );
+ EntityManager em = emf.getEntityManager( appInfo.getId() );
+
+ Fairy fairy = Fairy.create();
+
+ List<Entity> users = new ArrayList<Entity>( numUsers );
+
+ for (int i = 0; i < numUsers; i++) {
+
+ final Person person = fairy.person();
+ Entity userEntity = null;
+ try {
+ final Map<String, Object> userMap = new HashMap<String, Object>() {{
+ put( "username", person.username() );
+ put( "password", person.password() );
+ put( "email", person.email() );
+ put( "companyEmail", person.companyEmail() );
+ put( "dateOfBirth", person.dateOfBirth() );
+ put( "firstName", person.firstName() );
+ put( "lastName", person.lastName() );
+ put( "nationalIdentificationNumber", person.nationalIdentificationNumber() );
+ put( "telephoneNumber", person.telephoneNumber() );
+ put( "passportNumber", person.passportNumber() );
+ put( "address", person.getAddress() );
+ }};
+
+ userEntity = em.create( "user", userMap );
+ users.add( userEntity );
+
+ } catch (DuplicateUniquePropertyExistsException e) {
+ logger.error( "Dup user generated: " + person.username() );
+ continue;
+ } catch (Exception e) {
+ logger.error("Error creating user", e);
+ continue;
+ }
+
+ final Company company = person.getCompany();
+ try {
+ EntityRef ref = em.getAlias( "company", company.name() );
+ Entity companyEntity = (ref == null) ? null : em.get( ref );
+
+ // create company if it does not exist yet
+ if ( companyEntity == null ) {
+ final Map<String, Object> companyMap = new HashMap<String, Object>() {{
+ put( "name", company.name() );
+ put( "domain", company.domain() );
+ put( "email", company.email() );
+ put( "url", company.url() );
+ put( "vatIdentificationNumber", company.vatIdentificationNumber() );
+ }};
+ companyEntity = em.create( "company", companyMap );
+ } else {
+ logger.info("Company {} already exists", company.name());
+ }
+
+ em.createConnection( userEntity, "employer", companyEntity );
+
+ } catch (DuplicateUniquePropertyExistsException e) {
+ logger.error( "Dup company generated {} property={}", company.name(), e.getPropertyName() );
+ continue;
+ } catch (Exception e) {
+ logger.error("Error creating or connecting company", e);
+ continue;
+ }
+
+ try {
+ for (int j = 0; j < 5; j++) {
+ Activity activity = new Activity();
+ Activity.ActivityObject actor = new Activity.ActivityObject();
+ actor.setEntityType( "user" );
+ actor.setId( userEntity.getUuid().toString() );
+ activity.setActor( actor );
+ activity.setVerb( "POST" );
+ activity.setContent( "User " + person.username() + " generated a random string "
+ + RandomStringUtils.randomAlphanumeric( 5 ) );
+ em.createItemInCollection( userEntity, "activities", "activity", activity.getProperties() );
+ }
+
+ if (users.size() > 10) {
+ for (int j = 0; j < 5; j++) {
+ try {
+ em.createConnection( userEntity, "associate", users.get( (int) (Math.random() * users.size()) ) );
+ } catch (Exception e) {
+ logger.error( "Error connecting user to user: " + e.getMessage() );
+ }
+ }
+ }
+
+ } catch (Exception e) {
+ logger.error("Error creating activities", e);
+ continue;
+ }
- em.createItemInCollection( firstUserEntity, "activities", "activity", activity.getProperties() );
+ }
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a6e68a0a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
index 63187a4..3c427e1 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
@@ -116,7 +116,7 @@ public abstract class ToolBase {
public void printCliHelp( String message ) {
System.out.println( message );
HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp( "java -jar usergrid-tools-0.0.1-SNAPSHOT.jar " + getToolName(), createOptions() );
+ formatter.printHelp( "java -jar usergrid-tools-1.0.2.jar " + getToolName(), createOptions() );
System.exit( -1 );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a6e68a0a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
index a1e3f6b..446aa91 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
@@ -63,65 +63,23 @@ public class ExportAppTest {
// create app with some data
- OrganizationOwnerInfo orgInfo = setup.getMgmtSvc().createOwnerAndOrganization(
- "org_" + rand, "user_" + rand, rand.toUpperCase(), rand + "@example.com", rand );
-
- ApplicationInfo appInfo = setup.getMgmtSvc().createApplication(
- orgInfo.getOrganization().getUuid(), "app_" + rand );
-
- final EntityManager em = setup.getEmf().getEntityManager( appInfo.getId() );
+ String orgName = "org_" + rand;
+ String appName = "app_" + rand;
+
+ ExportDataCreator creator = new ExportDataCreator();
+ creator.startTool( new String[] {
+ "-organization", orgName,
+ "-application", appName,
+ "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort()
+ }, false);
- // create connected things
-
- final List<Entity> connectedThings = new ArrayList<Entity>();
- String connectedType = "connected_thing";
- em.createApplicationCollection(connectedType);
- for ( int j=0; j<NUM_CONNECTIONS; j++) {
- final String name = "connected_thing_" + j;
- connectedThings.add( em.create( connectedType, new HashMap<String, Object>() {{
- put( "name", name );
- }} ) );
- }
-
- // create collections of things, every other thing is connected to the connected things
-
- final AtomicInteger entitiesCount = new AtomicInteger(0);
- final AtomicInteger connectionCount = new AtomicInteger(0);
-
- ExecutorService execService = Executors.newFixedThreadPool( 50);
- final Scheduler scheduler = Schedulers.from( execService );
-
- for (int i = 0; i < NUM_COLLECTIONS; i++) {
-
- final String type = "thing_" + i;
- em.createApplicationCollection( type );
- connectionCount.getAndIncrement();
-
- for (int j = 0; j < NUM_ENTITIES; j++) {
- final String name = "thing_" + j;
- final Entity source = em.create(
- type, new HashMap<String, Object>() {{
- put( "name", name );
- }} );
- entitiesCount.getAndIncrement();
-
- for (Entity target : connectedThings) {
- em.createConnection( source, "has", target );
- connectionCount.getAndIncrement();
- }
- }
- }
-
- logger.info( "Done. Created {} entities and {} connections", entitiesCount.get(), connectionCount.get() );
-
long start = System.currentTimeMillis();
String directoryName = "target/export" + rand;
ExportApp exportApp = new ExportApp();
exportApp.startTool( new String[]{
- "-application", appInfo.getName(),
- "-readThreads", "100",
+ "-application", orgName + "/" + appName,
"-writeThreads", "100",
"-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
"-outputDir", directoryName
@@ -137,8 +95,7 @@ public class ExportAppTest {
File exportDir1 = new File(directoryName + "1");
exportApp.startTool( new String[]{
- "-application", appInfo.getName(),
- "-readThreads", "1",
+ "-application", orgName + "/" + appName,
"-writeThreads", "1",
"-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
"-outputDir", directoryName + "1"