You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/21 15:21:28 UTC
[37/38] git commit: Write,
index and read performance test modules for new Core Persistence API
and old Entity Manager API.
Write, index and read performance test modules for new Core Persistence API and old Entity Manager API.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d2cb960f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d2cb960f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d2cb960f
Branch: refs/heads/two-dot-o
Commit: d2cb960f40de016f1c8bbaa3a54e9fbc295e7e9e
Parents: e2f6fbf
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Mar 21 09:55:35 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Mar 21 09:55:35 2014 -0400
----------------------------------------------------------------------
.gitignore | 6 +
stack/corepersistence/perftest1/pom.xml | 97 ++++++
.../usergrid/persistence/UsergridBootstrap.java | 303 +++++++++++++++++
.../src/main/resources/log4j.properties | 44 +++
.../src/main/resources/project.properties | 1 +
.../main/resources/usergrid-test-context.xml | 39 +++
.../persistence/Usegrid1PerformanceIT.java | 304 ++++++++++++++++++
.../persistence/UsergridBootstrapTest.java | 56 ++++
stack/corepersistence/perftest2/pom.xml | 85 +++++
.../main/groovy/perftest2/CreateEntity.groovy | 100 ++++++
.../usergrid/persistence/CorePerformanceIT.java | 321 +++++++++++++++++++
.../index/guice/TestIndexModule.java | 30 ++
12 files changed, 1386 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 38c4045..c9f4468 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,9 @@ stack/corepersistence/nbactions.xml
stack/corepersistence/queryindex/nbactions.xml
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
+/stack/corepersistence/perftest2/src/main/resources/finefoods.txt
+/stack/corepersistence/perftest1/src/main/resources/finefoods.txt
+/stack/corepersistence/perftest2/nbactions.xml
+/stack/corepersistence/perftest1/nbactions.xml
+/stack/corepersistence/perftest1/src/main/resources/usergrid.properties
+/stack/corepersistence/perftest2/src/main/resources/usergrid.properties
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/pom.xml b/stack/corepersistence/perftest1/pom.xml
new file mode 100644
index 0000000..abee483
--- /dev/null
+++ b/stack/corepersistence/perftest1/pom.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>perftest1</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.compiler.source>1.7</maven.compiler.source>
+ <maven.compiler.target>1.7</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-core</artifactId>
+ <version>0.0.29-SNAPSHOT</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-test-utils</artifactId>
+ <version>0.0.29-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <artifactId>groovy-all</artifactId>
+ <groupId>org.codehaus.groovy</groupId>
+ <version>2.2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-tools</artifactId>
+ <version>0.0.29-SNAPSHOT</version>
+ <scope>test</scope>
+ <type>jar</type>
+ </dependency>
+ </dependencies>
+
+ <name>perftest1</name>
+
+ <build>
+ <plugins>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.3.2</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>2.6.0-01</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <groupId>org.codehaus.groovy</groupId>
+ <version>2.6.0-01</version>
+ <extensions>true</extensions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java b/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java
new file mode 100644
index 0000000..67f4af4
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.usergrid.cassandra.SchemaManager;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.StringUtils;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_APPLICATION;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_ORGANIZATION;
+
+
+/**
+ * Does the Spring magic to get bootstrap a Usergrid CassandraService, Entity Manager Factory and
+ * entity manager. Based on code from CassandraResource in the test-utils module.
+ * <p/>
+ * Note that for this resource to work properly, a project.properties file must be placed in the
+ * src/test/resources directory with the following stanza in the project pom's build section:
+ * <p/>
+ * <testResources> <testResource> <directory>src/test/resources</directory> <filtering>true</filtering> <includes>
+ * <include>**\/*.properties</include> <include>**\/*.xml</include> </includes> </testResource> </testResources>
+ * <p/>
+ * The following property expansion macro should be placed in this project.properties file:
+ * <p/>
+ * target.directory=${pom.build.directory}
+ */
+public class UsergridBootstrap { // is that a great classname or what!
+
+ public static final Logger LOG = LoggerFactory.getLogger( UsergridBootstrap.class );
+
+ public static final String PROPERTIES_FILE = "project.properties";
+
+ public static final String TARGET_DIRECTORY_KEY = "target.directory";
+
+ private static final Object lock = new Object();
+
+ private final File tempDir;
+ private final String schemaManagerName;
+
+ private boolean initialized = false;
+
+ private ConfigurableApplicationContext applicationContext;
+ private SchemaManager schemaManager;
+
+ private static UsergridBootstrap instance;
+ private Thread shutdown;
+
+
+ /**
+ * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the
+ * specified SchemaManager for Cassandra.
+ */
+ UsergridBootstrap( String schemaManagerName ) throws Throwable {
+ LOG.info( "Creating CassandraResource using {} for the ClassLoader.",
+ Thread.currentThread().getContextClassLoader() );
+
+ this.schemaManagerName = schemaManagerName;
+ try {
+ this.tempDir = getTempDirectory();
+ }
+ catch ( Exception e ) {
+ LOG.error( "Failed to create temporary directory for Cassandra instance.", e );
+ throw new RuntimeException( e );
+ }
+
+ before();
+ }
+
+
+ /**
+ * Gets the temporary directory created for this CassandraResource.
+ *
+ * @return the temporary directory
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public File getTemporaryDirectory() {
+ return tempDir;
+ }
+
+
+ /**
+ * Gets a bean from the application context.
+ *
+ * @param requiredType the type of the bean
+ * @param <T> the type of the bean
+ *
+ * @return the bean
+ */
+ public <T> T getBean( String name, Class<T> requiredType ) {
+ protect();
+ return applicationContext.getBean( name, requiredType );
+ }
+
+
+ /**
+ * Gets a bean from the application context.
+ *
+ * @param requiredType the type of the bean
+ * @param <T> the type of the bean
+ *
+ * @return the bean
+ */
+ public <T> T getBean( Class<T> requiredType ) {
+ protect();
+ return applicationContext.getBean( requiredType );
+ }
+
+
+ /**
+ * Gets whether this resource is ready to use.
+ *
+ * @return true if ready to use, false otherwise
+ */
+ public boolean isReady() {
+ return initialized;
+ }
+
+ /**
+ * Protects against IDE or command line runs of a unit test which when starting the test
+ * outside of the Suite will not start the resource. This makes sure the resource is
+ * automatically started on a usage attempt.
+ */
+ private void protect() {
+ if ( !isReady() ) {
+ try {
+ before();
+ }
+ catch ( Throwable t ) {
+ LOG.error( "Failed to start up Cassandra.", t );
+ throw new RuntimeException( t );
+ }
+ }
+ }
+
+
+ /**
+ * before TestSuite or test Class execution.
+ *
+ * @throws Throwable if we cannot start up Cassandra
+ */
+ protected void before() throws Throwable {
+
+ String[] locations = { "usergrid-test-context.xml" };
+ applicationContext = new ClassPathXmlApplicationContext( locations );
+
+ EntityManagerFactory emf = applicationContext.getBean( EntityManagerFactory.class );
+
+ String appName = StringUtils.lowerCase( DEFAULT_APPLICATION.contains( "/" )
+ ? DEFAULT_APPLICATION : DEFAULT_ORGANIZATION + "/" + DEFAULT_APPLICATION );
+
+ if ( emf.lookupApplication(appName) == null ) {
+ loadSchemaManager( schemaManagerName );
+ }
+
+ initialized = true;
+ LOG.info( "External Cassandra resource at is ready!");
+ lock.notifyAll();
+ }
+
+
+ /** Stops Cassandra after a TestSuite or test Class executes. */
+ protected synchronized void after() {
+
+ shutdown = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Thread.currentThread().sleep( 100L );
+ }
+ catch ( InterruptedException e ) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+
+ if ( schemaManager != null ) {
+ LOG.info( "Destroying schemaManager..." );
+ try {
+ schemaManager.destroy();
+ }
+ catch ( Exception e ) {
+ LOG.error( "Ignoring failures while dropping keyspaces: {}", e.getMessage() );
+ }
+
+ LOG.info( "SchemaManager destroyed..." );
+ }
+
+ applicationContext.stop();
+ LOG.info( "ApplicationContext stopped..." );
+
+ }
+ };
+
+ shutdown.start();
+ }
+
+
+ /**
+ * Loads the specified {@link SchemaManager} or the default manager if the manager
+ * name that is provided is null.
+ *
+ * @param schemaManagerName the name of the SchemaManager to load, or null
+ */
+ private void loadSchemaManager( String schemaManagerName ) {
+ if ( !applicationContext.isActive() ) {
+ LOG.info( "Restarting context..." );
+ applicationContext.refresh();
+ }
+
+ if ( schemaManagerName != null ) {
+ LOG.info( "Looking up SchemaManager impl: {}", schemaManagerName );
+ this.schemaManager = applicationContext.getBean( schemaManagerName, SchemaManager.class );
+ }
+ else {
+ LOG.info( "The SchemaManager is not specified - using the default SchemaManager impl" );
+ this.schemaManager = applicationContext.getBean( SchemaManager.class );
+ }
+
+ schemaManager.create();
+ schemaManager.populateBaseData();
+ }
+
+
+ /**
+ * Creates a new instance
+ *
+ * @param schemaManagerName the name of the schemaManager to use
+ *
+ * @return a new CassandraResource with possibly non-default ports
+ */
+ public static UsergridBootstrap newInstance( String schemaManagerName ) throws Throwable {
+ // Uncomment to test for Surefire Failures
+ // System.setSecurityManager( new NoExitSecurityManager( System.getSecurityManager() ) );
+
+ synchronized ( lock ) {
+ if ( instance != null ) {
+ return instance;
+ }
+
+ instance = new UsergridBootstrap( schemaManagerName );
+ LOG.info("Created a new instance of CassandraResource: {}", instance);
+ return instance;
+ }
+ }
+
+
+ public static UsergridBootstrap newInstance() throws Throwable {
+ return newInstance( null );
+ }
+
+
+ /**
+ * Uses a project.properties file that Maven does substitution on to to replace the value of
+ * a property with the path to the Maven build directory (a.k.a. target). It then uses this
+ * path to generate a random String which it uses to append a path component to so a unique
+ * directory is selected. If already present it's deleted, then the directory is created.
+ *
+ * @return a unique temporary directory
+ *
+ * @throws IOException if we cannot access the properties file
+ */
+ public static File getTempDirectory() throws IOException {
+ File tmpdir;
+ Properties props = new Properties();
+ props.load( ClassLoader.getSystemResourceAsStream( PROPERTIES_FILE ) );
+ File basedir = new File( ( String ) props.get( TARGET_DIRECTORY_KEY ) );
+ String comp = RandomStringUtils.randomAlphanumeric( 7 );
+ tmpdir = new File( basedir, comp );
+
+ if ( tmpdir.exists() ) {
+ LOG.info( "Deleting directory: {}", tmpdir );
+ FileUtils.forceDelete( tmpdir );
+ }
+ else {
+ LOG.info( "Creating temporary directory: {}", tmpdir );
+ FileUtils.forceMkdir( tmpdir );
+ }
+
+ return tmpdir;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/resources/log4j.properties b/stack/corepersistence/perftest1/src/main/resources/log4j.properties
new file mode 100644
index 0000000..fab8d7b
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/resources/log4j.properties
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# for production, you should probably set the root to INFO
+# and the pattern to %c instead of %l. (%l is slower.)
+
+# output messages into a rolling log file as well as stdout
+log4j.rootLogger=INFO,stdout
+
+# stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+#log4j.appender.stdout.layout=org.apache.log4j.SimpleLayout
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
+
+log4j.category.org.apache=ERROR, stdout
+
+log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.DaoUtils=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.ConnectionRefImpl=WARN, stdout
+log4j.logger.me.prettyprint.cassandra.hector.TimingLogger=WARN, stdout
+log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
+log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
+#log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
+
+
+#log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/resources/project.properties b/stack/corepersistence/perftest1/src/main/resources/project.properties
new file mode 100644
index 0000000..c0bf364
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/resources/project.properties
@@ -0,0 +1 @@
+target.directory=${project.build.directory}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml b/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml
new file mode 100644
index 0000000..9bdabe6
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xmlns:context="http://www.springframework.org/schema/context" xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:hz="http://www.hazelcast.com/schema/config" xmlns:aop="http://www.springframework.org/schema/aop"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
+ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.1.xsd
+ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd">
+
+ <!-- configure our test properties -->
+ <bean id="properties"
+ class="org.springframework.beans.factory.config.PropertiesFactoryBean">
+ <property name="singleton" value="true" />
+ <property name="ignoreResourceNotFound" value="true" />
+ <property name="locations">
+ <list>
+ <value>classpath:/usergrid-default.properties</value>
+ <value>classpath:/usergrid.properties</value>
+ <value>classpath:/usergrid-scheduler-test.properties</value>
+ </list>
+ </property>
+ </bean>
+
+ <import resource="classpath:/usergrid-core-context.xml"/>
+
+ <bean id="setup" class="org.apache.usergrid.persistence.cassandra.Setup">
+ <constructor-arg ref="entityManagerFactory"/>
+ <constructor-arg ref="cassandraService"/>
+ </bean>
+
+ <!-- Refer to a named schemaManager from the DataControl annotation thusly -->
+ <bean id="coreManager" class="org.apache.usergrid.persistence.CoreSchemaManager">
+ <constructor-arg ref="setup"/>
+ <constructor-arg ref="cassandraCluster"/>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
new file mode 100644
index 0000000..af47a8c
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.NumberUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class Usegrid1PerformanceIT {
+ private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceIT.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityManagerFactory emf;
+
+
+ public Usegrid1PerformanceIT() throws Throwable {
+ emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
+ }
+
+
+ public EntityManagerFactory getEmf() {
+ return emf;
+ }
+
+
+ @Ignore
+ @Test
+ public void loadAndReadData() throws Exception {
+
+ log.info("Start Data Load");
+ List<UUID> apps = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( apps );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( apps );
+
+ }
+
+
+ private List<UUID> loadData() throws Exception {
+
+ long time = new Date().getTime();
+
+ List<UUID> apps = new ArrayList<UUID>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ UUID appId = getEmf().createApplication(
+ "testorg-" + RandomStringUtils.randomAlphanumeric(6),
+ "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+ apps.add( appId );
+
+ Thread t = new Thread( new DataLoader( appId ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return apps;
+ }
+
+
+ private void readData( List<UUID> apps ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( UUID app : apps ) {
+
+ Thread t = new Thread( new DataReader( app ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+
+ private class DataReader implements Runnable {
+ UUID app;
+
+ public DataReader( UUID app ) {
+ this.app = app;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ UUID appId = app;
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ return;
+ }
+
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ log.info("Read {} reviews in {}", count, appId );
+ return;
+ }
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {}", count, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ UUID app;
+
+ public DataLoader( UUID scope ) {
+ this.app = scope;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Map<String, Object> currentEntityMap = new HashMap<String, Object>();
+
+ UUID appId = app;
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ Entity entity = em.create("review", currentEntityMap );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", entity.getUuid());
+ log.info("---");
+ }
+
+ // create the next entity
+ currentEntityMap = new HashMap<String, Object>();
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {}", count, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ currentEntityMap.put( name, Double.parseDouble(value));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ currentEntityMap.put( name, Long.parseLong(value));
+
+ } else {
+ currentEntityMap.put( name, value.toString() );
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+ }
+ }
+
+
+ public void runSelectedQueries( List<UUID> apps ) throws Exception {
+
+ for ( UUID app : apps ) {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ query(em, "product_productid = 'B006K2ZZ7K'") ;
+ query(em, "review_profilename = 'Twoapennything'") ;
+ query(em, "review_profilename contains 'Natalia'") ;
+ query(em, "review_profilename contains 'Patrick'") ;
+ query(em, "review_time = 1342051200") ;
+ query(em, "review_time > 1342051200") ;
+ query(em, "review_score > 0");
+ query(em, "review_score > 2");
+ query(em, "review_score > 3");
+ query(em, "review_score > 4");
+ query(em, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityManager em, String query ) throws Exception {
+ Query q = Query.fromQL(query) ;
+ Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java
new file mode 100644
index 0000000..dfdac5d
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java
@@ -0,0 +1,56 @@
+
+package org.apache.usergrid.persistence;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class UsergridBootstrapTest {
+ private static final Logger log = LoggerFactory.getLogger(UsergridBootstrapTest.class);
+
+ @Ignore // just an example and not included in performance test
+ @Test
+ public void testBasicOperation() throws Throwable {
+
+ UsergridBootstrap sbf = UsergridBootstrap.newInstance();
+ EntityManagerFactory emf = sbf.getBean( EntityManagerFactory.class );
+
+ UUID appId = emf.createApplication(
+ "testorg-" + RandomStringUtils.randomAlphanumeric(6),
+ "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+ EntityManager em = emf.getEntityManager( appId );
+
+ Map<String, Object> map = new HashMap<String, Object>() {{
+ put("name", RandomStringUtils.randomAlphanumeric(6));
+ put("timestamp", new Date().getTime());
+ }};
+ Entity entity = em.create( "testentity", map );
+ String name = (String)entity.getProperty( "name" );
+
+ Entity got = em.get( entity.getUuid() );
+ assertNotNull(got);
+ String returnedName = (String)got.getProperty("name");
+ long timestamp = (long)got.getProperty("timestamp");
+ assertEquals( name, returnedName );
+ log.info(">>>> Got back name={} : time={}", returnedName, timestamp);
+
+ Query query = Query.fromQL("name = '" + name + "'");
+ Entity found = em.searchCollection( em.getApplication(), "testentities", query ).getEntity();
+ assertNotNull(found);
+ returnedName = (String)found.getProperty("name");
+ timestamp = (long)found.getProperty("timestamp");
+ assertEquals( name, returnedName );
+ log.info(">>>> Found name={} : time={}", returnedName, timestamp);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/pom.xml b/stack/corepersistence/perftest2/pom.xml
new file mode 100644
index 0000000..be1a457
--- /dev/null
+++ b/stack/corepersistence/perftest2/pom.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>perftest2</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.compiler.source>1.7</maven.compiler.source>
+ <maven.compiler.target>1.7</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>queryindex</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <artifactId>groovy-all</artifactId>
+ <groupId>org.codehaus.groovy</groupId>
+ <version>2.2.2</version>
+ </dependency>
+
+ </dependencies>
+
+ <name>perftest2</name>
+
+ <build>
+ <plugins>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.3.2</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>2.6.0-01</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <groupId>org.codehaus.groovy</groupId>
+ <version>2.6.0-01</version>
+ <extensions>true</extensions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy b/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy
new file mode 100644
index 0000000..d1a6477
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package perftest2;
+
+import com.google.inject.Injector
+import com.netflix.config.ConfigurationManager
+import org.apache.commons.lang3.RandomStringUtils
+import org.apache.usergrid.persistence.collection.CollectionScope
+import org.apache.usergrid.persistence.collection.EntityCollectionManager
+import com.google.inject.Guice
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl
+import org.apache.usergrid.persistence.collection.migration.MigrationManager
+import org.apache.usergrid.persistence.collection.util.EntityUtils
+import org.apache.usergrid.persistence.index.EntityCollectionIndex
+import org.apache.usergrid.persistence.index.guice.IndexModule
+import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory
+import org.apache.usergrid.persistence.model.entity.Entity
+import org.apache.usergrid.persistence.model.entity.Id
+import org.apache.usergrid.persistence.model.entity.SimpleId
+import org.apache.usergrid.persistence.model.field.LongField
+import org.apache.usergrid.persistence.model.field.StringField
+import org.apache.usergrid.persistence.model.util.UUIDGenerator
+import org.apache.usergrid.persistence.query.Query
+import org.apache.usergrid.persistence.query.Results
+
+
+/**
+ * Just an example and not included in performance test.
+ */
+public class CreateEntity {
+
+ public static void main( String[] args ) {
+
+ // setup core persistence
+ ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" )
+ Injector injector = Guice.createInjector( new IndexModule() )
+
+ // only on first run
+ //MigrationManager m = injector.getInstance( MigrationManager.class )
+ //m.migrate()
+
+ EntityCollectionManagerFactory ecmf =
+ injector.getInstance( EntityCollectionManagerFactory.class )
+ EntityCollectionIndexFactory ecif =
+ injector.getInstance( EntityCollectionIndexFactory.class )
+
+ // create scope
+ Id appId = new SimpleId("entitytest")
+ Id orgId = new SimpleId("usergrid")
+ CollectionScope scope = new CollectionScopeImpl( appId, orgId, "testentities" )
+
+ // entity manager and index interfaces
+ EntityCollectionManager ecm = ecmf.createCollectionManager( scope )
+ EntityCollectionIndex eci = ecif.createCollectionIndex( scope )
+
+ // create entity
+ Entity entity = new Entity(new SimpleId(UUIDGenerator.newTimeUUID(), "testentity"))
+ String name = RandomStringUtils.randomAlphanumeric(10)
+ entity.setField( new StringField("name", name))
+ entity.setField( new LongField("timestamp", new Date().getTime()))
+
+ // write and index entity
+ ecm.write( entity ).toBlockingObservable().last()
+ eci.index( entity )
+
+ eci.refresh()
+
+ ecm = ecmf.createCollectionManager( scope )
+ eci = ecif.createCollectionIndex( scope )
+
+ // get back entity
+ def got = ecm.load( entity.getId() ).toBlockingObservable().last()
+
+ def returnedName = got.getField("name").getValue()
+ def timestamp = got.getField("timestamp").getValue()
+ println ">>>> Got back name=${returnedName} : time=${timestamp}"
+
+ // search for entity
+ def found = eci.execute( Query.fromQL( "name = '${name}'".toString() ) ).getEntities().get(0)
+
+ returnedName = found.getField("name").getValue()
+ timestamp = found.getField("timestamp").getValue()
+ println ">>>> Found name=${returnedName} : time=${timestamp}"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
new file mode 100644
index 0000000..8536fca
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.netflix.config.ConfigurationManager;
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.index.EntityCollectionIndex;
+import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.query.Query;
+import org.apache.usergrid.persistence.query.Results;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class CorePerformanceIT {
+ private static final Logger log = LoggerFactory.getLogger(CorePerformanceIT.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityCollectionManagerFactory ecmf;
+ private final EntityCollectionIndexFactory ecif;
+
+ public CorePerformanceIT() {
+ Injector injector = Guice.createInjector( new TestIndexModule() );
+ ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
+ ecif = injector.getInstance( EntityCollectionIndexFactory.class );
+ }
+
+ @Ignore
+ @Test
+ public void loadAndReadData() throws IOException, InterruptedException {
+
+ ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
+
+ // only on first run
+ //MigrationManager m = injector.getInstance( MigrationManager.class )
+ //m.migrate()
+
+ log.info("Start Data Load");
+ List<CollectionScope> scopes = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( scopes );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( scopes );
+
+ }
+
+
+ private List<CollectionScope> loadData() throws InterruptedException {
+
+ long time = new Date().getTime();
+
+ List<CollectionScope> scopes = new ArrayList<CollectionScope>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ String orgName = "org-" + i + "-" + time;
+ final Id orgId = new SimpleId(orgName);
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ String appName = "app-" + j + "-" + time;
+ final Id appId = new SimpleId(appName);
+
+ CollectionScope scope = new CollectionScopeImpl( orgId, appId, "reviews" );
+ scopes.add( scope );
+
+ Thread t = new Thread( new DataLoader( scope ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return scopes;
+ }
+
+
+ private void readData( List<CollectionScope> scopes ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( CollectionScope scope : scopes ) {
+
+ Thread t = new Thread( new DataReader( scope ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+ /**
+ * @return the ecmf
+ */
+ public EntityCollectionManagerFactory getEcmf() {
+ return ecmf;
+ }
+
+ /**
+ * @return the ecif
+ */
+ public EntityCollectionIndexFactory getEcif() {
+ return ecif;
+ }
+
+
+ private class DataReader implements Runnable {
+ CollectionScope scope;
+
+ public DataReader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {} / {} ", count, orgId, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ CollectionScope scope;
+
+ public DataLoader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Entity current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ ecm.write( current ).toBlockingObservable().last();
+ eci.index( current );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", current.getId());
+ log.info("---");
+ }
+
+ // create the next entity
+ current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {} / {} ", count, orgId, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ current.setField( new DoubleField( name, Double.parseDouble(value)));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ current.setField( new LongField( name, Long.parseLong(value)));
+
+ } else {
+ current.setField( new StringField( name, value.toString() ));
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+
+ eci.refresh();
+ }
+ }
+
+
+ public void runSelectedQueries( List<CollectionScope> scopes ) {
+
+ for ( CollectionScope scope : scopes ) {
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ // TODO: come up with more and more complex queries for CorePerformanceIT
+
+ query(eci, "product_productid = 'B006K2ZZ7K'") ;
+ query(eci, "review_profilename = 'Twoapennything'") ;
+ query(eci, "review_profilename contains 'Natalia'") ;
+ query(eci, "review_profilename contains 'Patrick'") ;
+ query(eci, "review_time = 1342051200") ;
+ query(eci, "review_time > 1342051200") ;
+ query(eci, "review_score > 0");
+ query(eci, "review_score > 2");
+ query(eci, "review_score > 3");
+ query(eci, "review_score > 4");
+ query(eci, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityCollectionIndex eci, String query ) {;
+ Query q = Query.fromQL(query) ;
+ Results results = eci.execute( q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
new file mode 100644
index 0000000..3c2390b
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.index.guice;
+
+import org.apache.usergrid.persistence.collection.guice.TestModule;
+
+
+public class TestIndexModule extends TestModule {
+
+ @Override
+ protected void configure() {
+ install( new IndexModule() );
+ }
+}