You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/24 16:20:48 UTC
[01/18] git commit: JUnit test version of the code I've been using to
test Core Persistence load and read performance.
Repository: incubator-usergrid
Updated Branches:
refs/heads/hystrix-integration 457528f50 -> b1d12a171
refs/heads/two-dot-o c4f9d0f20 -> fc105970f
JUnit test version of the code I've been using to test Core Persistence load and read performance.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b3b46e4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b3b46e4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b3b46e4c
Branch: refs/heads/hystrix-integration
Commit: b3b46e4cc6c77a9c92236863dcc2f7b711e3e4b1
Parents: 070f15b
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Mar 19 15:19:45 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Mar 19 15:19:45 2014 -0400
----------------------------------------------------------------------
.../index/impl/CorePerformanceIT.java | 304 +++++++++++++++++++
1 file changed, 304 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b3b46e4c/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
new file mode 100644
index 0000000..b368b92
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.netflix.config.ConfigurationManager;
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.index.EntityCollectionIndex;
+import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.query.Query;
+import org.apache.usergrid.persistence.query.Results;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make CorePerformanceIT configurable, add CHOP markup.
+ */
+public class CorePerformanceIT {
+ private static final Logger log = LoggerFactory.getLogger(CorePerformanceIT.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ static EntityCollectionManagerFactory ecmf;
+ static EntityCollectionIndexFactory ecif ;
+
+
+ @Ignore
+ @Test
+ public void loadAndReadData( String[] args ) throws IOException, InterruptedException {
+
+ ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
+ Injector injector = Guice.createInjector( new TestIndexModule() );
+
+ // only on first run
+ //MigrationManager m = injector.getInstance( MigrationManager.class )
+ //m.migrate()
+
+ ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
+ ecif = injector.getInstance( EntityCollectionIndexFactory.class );
+
+ log.info("Start Data Load");
+ List<CollectionScope> scopes = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( scopes );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( scopes );
+
+ }
+
+
+ private List<CollectionScope> loadData() throws InterruptedException {
+
+ long time = new Date().getTime();
+
+ List<CollectionScope> scopes = new ArrayList<CollectionScope>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ String orgName = "org-${i}-${time}";
+ final Id orgId = new SimpleId(orgName);
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ String appName = "app-${j}-${time}";
+ final Id appId = new SimpleId(appName);
+
+ CollectionScope scope = new CollectionScopeImpl( orgId, appId, "reviews" );
+ scopes.add( scope );
+
+// def scopeFile = new File("/home/ubuntu/scopes.txt")
+// scopeFile.append("Created ${orgId}, ${appId}\n")
+
+ Thread t = new Thread( new DataLoader( scope ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return scopes;
+ }
+
+
+ private void readData( List<CollectionScope> scopes ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( CollectionScope scope : scopes ) {
+
+ Thread t = new Thread( new DataReader( scope ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+
+ static class DataReader implements Runnable {
+ CollectionScope scope;
+
+ public DataReader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
+ EntityCollectionManager ecm = ecmf.createCollectionManager( scope );
+ EntityCollectionIndex eci = ecif.createCollectionIndex( scope );
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read ${count} reviews in ${orgId} ${appId}");
+ }
+ }
+ }
+
+
+ static class DataLoader implements Runnable {
+ CollectionScope scope;
+
+ public DataLoader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ EntityCollectionManager ecm = ecmf.createCollectionManager( scope );
+ EntityCollectionIndex eci = ecif.createCollectionIndex( scope );
+
+ FileReader fr;
+ try {
+ fr = new FileReader("../../resources/finefoods.txt");
+ } catch (FileNotFoundException ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ BufferedReader br = new BufferedReader(fr);
+ String s = null;
+
+ // create the first entry
+ Entity current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ ecm.write( current ).toBlockingObservable().last();
+ eci.index( current );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for ${current.getId()}");
+ log.info("---");
+ }
+
+ // create the next entity
+ current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed ${count} reviews in ${orgId} ${appId}");
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing ${name} = ${value}");
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ current.setField( new DoubleField( name, Double.parseDouble(value)));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ current.setField( new LongField( name, Long.parseLong(value)));
+
+ } else {
+ current.setField( new StringField( name, value.toString() ));
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+
+ eci.refresh();
+ }
+ }
+
+
+ public void runSelectedQueries( List<CollectionScope> scopes ) {
+
+ for ( CollectionScope scope : scopes ) {
+
+ EntityCollectionManager ecm = ecmf.createCollectionManager( scope );
+ EntityCollectionIndex eci = ecif.createCollectionIndex( scope );
+
+ // TODO: come up with more and more complex queries for CorePerformanceIT
+
+ query(eci, "product_productid = 'B006K2ZZ7K'") ;
+ query(eci, "review_profilename = 'Twoapennything'") ;
+ query(eci, "review_profilename contains 'Natalia'") ;
+ query(eci, "review_profilename contains 'Patrick'") ;
+ query(eci, "review_time = 1342051200") ;
+ query(eci, "review_time > 1342051200") ;
+ query(eci, "review_score > 0");
+ query(eci, "review_score > 2");
+ query(eci, "review_score > 3");
+ query(eci, "review_score > 4");
+ query(eci, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityCollectionIndex eci, String query ) {;
+ Query q = Query.fromQL(query) ;
+ Results results = eci.execute( q );
+ log.info("${q.getQl()}: ${results.getIds().size()}");
+ }
+
+}
[12/18] git commit: Dependency and other cleanup.
Posted by sn...@apache.org.
Dependency and other cleanup.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e1b4cffa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e1b4cffa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e1b4cffa
Branch: refs/heads/hystrix-integration
Commit: e1b4cffafaa6f8350cff78885c47e977f9e14cba
Parents: 9d6ed41
Author: Dave Johnson <dm...@apigee.com>
Authored: Sat Mar 22 18:15:51 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sat Mar 22 18:15:51 2014 -0400
----------------------------------------------------------------------
stack/corepersistence/perftest1/pom.xml | 9 ++---
.../persistence/Usergrid1PerformanceTest.java | 36 +++++++++++---------
2 files changed, 22 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1b4cffa/stack/corepersistence/perftest1/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/pom.xml b/stack/corepersistence/perftest1/pom.xml
index 0e380b1..0307bde 100644
--- a/stack/corepersistence/perftest1/pom.xml
+++ b/stack/corepersistence/perftest1/pom.xml
@@ -31,11 +31,6 @@
</dependency>
<dependency>
- <artifactId>groovy-all</artifactId>
- <groupId>org.codehaus.groovy</groupId>
- <version>2.2.2</version>
- </dependency>
- <dependency>
<groupId>org.apache.usergrid</groupId>
<artifactId>usergrid-tools</artifactId>
<version>0.0.29-SNAPSHOT</version>
@@ -49,7 +44,7 @@
<build>
<plugins>
- <plugin>
+<!-- <plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<dependencies>
@@ -69,7 +64,7 @@
<groupId>org.codehaus.groovy</groupId>
<version>2.6.0-01</version>
<extensions>true</extensions>
- </plugin>
+ </plugin>-->
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1b4cffa/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
index a818199..7aee702 100644
--- a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
@@ -36,8 +36,8 @@ import org.slf4j.LoggerFactory;
/**
* TODO: make configurable, add CHOP markup.
*/
-public class Usegrid1PerformanceTest {
- private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceTest.class);
+public class Usergrid1PerformanceTest {
+ private static final Logger log = LoggerFactory.getLogger(Usergrid1PerformanceTest.class);
// max entities we will write and read
static int maxEntities = Integer.MAX_VALUE;
@@ -53,7 +53,7 @@ public class Usegrid1PerformanceTest {
private final EntityManagerFactory emf;
- public Usegrid1PerformanceTest() throws Throwable {
+ public Usergrid1PerformanceTest() throws Throwable {
emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
}
@@ -66,13 +66,13 @@ public class Usegrid1PerformanceTest {
@Test
public void loadAndReadData() throws Exception {
- log.info("Start Data Load");
+ log("Start Data Load");
List<UUID> apps = loadData();
- log.info("Finish Data Load");
+ log("Finish Data Load");
- log.info("Start Data Read");
+ log("Start Data Read");
readData( apps );
- log.info("Finish Data Read");
+ log("Finish Data Read");
runSelectedQueries( apps );
@@ -167,13 +167,13 @@ public class Usegrid1PerformanceTest {
results = em.searchCollection( em.getApplicationRef(), "reviews", query );
} catch (Exception ex) {
log.error("Error on search, aborting", ex);
- log.info("Read {} reviews in {}", count, appId );
+ log( String.format("Read %d reviews in %s", count, appId) );
return;
}
results.getEntities(); // cause retrieval from Cassanda;
count += results.size();
- log.info("Read {} reviews in {}", count, appId );
+ log( String.format("Read %d reviews in %s", count, appId.toString()) );
}
}
}
@@ -223,8 +223,8 @@ public class Usegrid1PerformanceTest {
Entity entity = em.create("review", currentEntityMap );
if ( maxEntities < 20 ) {
- log.info("Index written for {}", entity.getUuid());
- log.info("---");
+ log( String.format("Index written for %s", entity.getUuid().toString()));
+ log("---");
}
// create the next entity
@@ -232,7 +232,7 @@ public class Usegrid1PerformanceTest {
count++;
if (count % 100000 == 0) {
- log.info("Indexed {} reviews in {}", count, appId );
+ log( String.format("Indexed %d reviews in %s", count, appId.toString()) );
}
continue;
}
@@ -242,7 +242,7 @@ public class Usegrid1PerformanceTest {
String value = s.substring( s.indexOf(":") + 1 ).trim();
if ( maxEntities < 20 ) {
- log.info("Indexing {} = {}", name, value);
+ log( String.format("Indexing %s = %s", name, value));
}
if ( NumberUtils.isNumber(value) && value.contains(".")) {
@@ -256,7 +256,7 @@ public class Usegrid1PerformanceTest {
}
} catch ( Exception e ) {
- log.info("Error on line " + count);
+ log("Error on line " + count);
}
}
@@ -293,10 +293,14 @@ public class Usegrid1PerformanceTest {
}
}
- public static void query( EntityManager em, String query ) throws Exception {
+ public void query( EntityManager em, String query ) throws Exception {
Query q = Query.fromQL(query) ;
Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
- log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ log( String.format("size = %d returned from query %s", results.size(), q.getQl()) );
}
+ private void log( String s ) {
+ //log.info(s);
+ System.out.println( System.currentTimeMillis() + ": " + s );
+ }
}
[09/18] git commit: Merge remote-tracking branch 'origin/two-dot-o'
into hystrix-integration
Posted by sn...@apache.org.
Merge remote-tracking branch 'origin/two-dot-o' into hystrix-integration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/58f7f436
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/58f7f436
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/58f7f436
Branch: refs/heads/hystrix-integration
Commit: 58f7f43642826c06f7aace698c6088a35186a6a0
Parents: 457528f 9d6ed41
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 21 10:15:31 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 21 10:15:31 2014 -0700
----------------------------------------------------------------------
.gitignore | 6 +
stack/awscluster/README.md | 31 +-
stack/corepersistence/perftest1/pom.xml | 76 +++++
.../usergrid/persistence/UsergridBootstrap.java | 303 ++++++++++++++++++
.../src/main/resources/log4j.properties | 44 +++
.../src/main/resources/project.properties | 1 +
.../main/resources/usergrid-test-context.xml | 39 +++
.../persistence/Usergrid1PerformanceTest.java | 302 ++++++++++++++++++
.../persistence/UsergridBootstrapTest.java | 56 ++++
stack/corepersistence/perftest2/pom.xml | 64 ++++
.../main/groovy/perftest2/CreateEntity.groovy | 100 ++++++
.../persistence/CorePerformanceTest.java | 318 +++++++++++++++++++
.../index/guice/TestIndexModule.java | 30 ++
.../index/impl/EsEntityCollectionIndex.java | 13 +-
.../index/impl/CorePerformanceIT.java | 304 ++++++++++++++++++
stack/pom.xml | 3 +
16 files changed, 1666 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
[13/18] git commit: Merge branch 'two-dot-o' of
https://github.com/usergrid/usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of https://github.com/usergrid/usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c4f9d0f2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c4f9d0f2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c4f9d0f2
Branch: refs/heads/hystrix-integration
Commit: c4f9d0f20ee4c342f7fec643f4eca35d2c384518
Parents: e1b4cff a705630
Author: Dave Johnson <dm...@apigee.com>
Authored: Sat Mar 22 18:16:08 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sat Mar 22 18:16:08 2014 -0400
----------------------------------------------------------------------
.../src/main/groovy/wait_for_instances.groovy | 2 +-
.../graph/consistency/AsyncProcessor.java | 14 +-
.../graph/consistency/AsyncProcessorImpl.java | 94 ++++++-
.../graph/consistency/TimeService.java | 33 +++
.../persistence/graph/guice/GraphModule.java | 7 +-
.../graph/consistency/AsyncProcessorTest.java | 273 +++++++++++++++++++
6 files changed, 409 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[06/18] git commit: Minor fixes, clean-ups.
Posted by sn...@apache.org.
Minor fixes, clean-ups.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4cf38be6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4cf38be6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4cf38be6
Branch: refs/heads/hystrix-integration
Commit: 4cf38be6d8401bef360e6b13cd2ab0360f51683d
Parents: d2cb960
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Mar 21 10:16:12 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Mar 21 10:16:12 2014 -0400
----------------------------------------------------------------------
stack/corepersistence/perftest1/pom.xml | 21 --
.../persistence/Usegrid1PerformanceIT.java | 304 ------------------
.../persistence/Usegrid1PerformanceTest.java | 302 +++++++++++++++++
stack/corepersistence/perftest2/pom.xml | 21 --
.../usergrid/persistence/CorePerformanceIT.java | 321 -------------------
.../persistence/CorePerformanceTest.java | 318 ++++++++++++++++++
6 files changed, 620 insertions(+), 667 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest1/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/pom.xml b/stack/corepersistence/perftest1/pom.xml
index abee483..0e380b1 100644
--- a/stack/corepersistence/perftest1/pom.xml
+++ b/stack/corepersistence/perftest1/pom.xml
@@ -71,27 +71,6 @@
<extensions>true</extensions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
- <overWriteReleases>false</overWriteReleases>
- <overWriteSnapshots>false</overWriteSnapshots>
- <overWriteIfNewer>true</overWriteIfNewer>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
</plugins>
</build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
deleted file mode 100644
index af47a8c..0000000
--- a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.commons.lang.NumberUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * TODO: make configurable, add CHOP markup.
- */
-public class Usegrid1PerformanceIT {
- private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceIT.class);
-
- // max entities we will write and read
- static int maxEntities = Integer.MAX_VALUE;
-
- // each app will get all data
- static int orgCount = 2;
- static int appCount = 5 ;
-
- // number of threads = orgCount x appCount
-
- // total number of records = orgCount x appCount x numRecords
-
- private final EntityManagerFactory emf;
-
-
- public Usegrid1PerformanceIT() throws Throwable {
- emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
- }
-
-
- public EntityManagerFactory getEmf() {
- return emf;
- }
-
-
- @Ignore
- @Test
- public void loadAndReadData() throws Exception {
-
- log.info("Start Data Load");
- List<UUID> apps = loadData();
- log.info("Finish Data Load");
-
- log.info("Start Data Read");
- readData( apps );
- log.info("Finish Data Read");
-
- runSelectedQueries( apps );
-
- }
-
-
- private List<UUID> loadData() throws Exception {
-
- long time = new Date().getTime();
-
- List<UUID> apps = new ArrayList<UUID>();
- List<Thread> threads = new ArrayList<Thread>();
-
- for ( int i=0; i<orgCount; i++ ) {
-
- for ( int j=0; j<appCount; j++ ) {
-
- UUID appId = getEmf().createApplication(
- "testorg-" + RandomStringUtils.randomAlphanumeric(6),
- "testapp-" + RandomStringUtils.randomAlphanumeric(6));
-
- apps.add( appId );
-
- Thread t = new Thread( new DataLoader( appId ));
- t.start();
- threads.add(t);
- }
- }
-
- // wait for indexing to end
- for ( Thread t : threads ) {
- t.join();
- }
-
- return apps;
- }
-
-
- private void readData( List<UUID> apps ) throws InterruptedException {
-
- List<Thread> threads = new ArrayList<Thread>();
- for ( UUID app : apps ) {
-
- Thread t = new Thread( new DataReader( app ));
- t.start();
- threads.add(t);
- }
-
- // wait for reading to end
- for ( Thread t : threads ) {
- t.join();
- }
- }
-
-
- private class DataReader implements Runnable {
- UUID app;
-
- public DataReader( UUID app ) {
- this.app = app;
- }
-
- public void run() {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- UUID appId = app;
-
- Query query = Query.fromQL( "review_score > 0"); // get all reviews;
- query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
-
- Results results;
- try {
- results = em.searchCollection( em.getApplicationRef(), "reviews", query );
- } catch (Exception ex) {
- log.error("Error on search, aborting", ex);
- return;
- }
-
- results.getEntities(); // cause retrieval from Cassandra
- int count = results.size();
-
- while ( results.hasCursor() && count < maxEntities ) {
- query.setCursor( results.getCursor() ) ;
- try {
- results = em.searchCollection( em.getApplicationRef(), "reviews", query );
- } catch (Exception ex) {
- log.error("Error on search, aborting", ex);
- log.info("Read {} reviews in {}", count, appId );
- return;
- }
- results.getEntities(); // cause retrieval from Cassanda;
- count += results.size();
-
- log.info("Read {} reviews in {}", count, appId );
- }
- }
- }
-
-
- private class DataLoader implements Runnable {
- UUID app;
-
- public DataLoader( UUID scope ) {
- this.app = scope;
- }
-
- public void run() {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- BufferedReader br;
- try {
- InputStreamReader isr = new InputStreamReader(
- getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
- br = new BufferedReader(isr);
- } catch (Exception ex) {
- throw new RuntimeException("Error opening file", ex);
- }
- String s = null;
-
- // create the first entry
- Map<String, Object> currentEntityMap = new HashMap<String, Object>();
-
- UUID appId = app;
-
- int count = 0;
- try {
- while ( (s = br.readLine()) != null && count < maxEntities ) {
-
- try {
-
- if ( s.trim().equals("")) { // then we are at end of a record
-
- // write and index current entity
- Entity entity = em.create("review", currentEntityMap );
-
- if ( maxEntities < 20 ) {
- log.info("Index written for {}", entity.getUuid());
- log.info("---");
- }
-
- // create the next entity
- currentEntityMap = new HashMap<String, Object>();
-
- count++;
- if (count % 100000 == 0) {
- log.info("Indexed {} reviews in {}", count, appId );
- }
- continue;
- }
-
- // process a field
- String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
- String value = s.substring( s.indexOf(":") + 1 ).trim();
-
- if ( maxEntities < 20 ) {
- log.info("Indexing {} = {}", name, value);
- }
-
- if ( NumberUtils.isNumber(value) && value.contains(".")) {
- currentEntityMap.put( name, Double.parseDouble(value));
-
- } else if ( NumberUtils.isNumber(value) ) {
- currentEntityMap.put( name, Long.parseLong(value));
-
- } else {
- currentEntityMap.put( name, value.toString() );
- }
-
- } catch ( Exception e ) {
- log.info("Error on line " + count);
- }
- }
-
- } catch (IOException ex) {
- throw new RuntimeException("Error reading file", ex);
- }
- }
- }
-
-
- public void runSelectedQueries( List<UUID> apps ) throws Exception {
-
- for ( UUID app : apps ) {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- query(em, "product_productid = 'B006K2ZZ7K'") ;
- query(em, "review_profilename = 'Twoapennything'") ;
- query(em, "review_profilename contains 'Natalia'") ;
- query(em, "review_profilename contains 'Patrick'") ;
- query(em, "review_time = 1342051200") ;
- query(em, "review_time > 1342051200") ;
- query(em, "review_score > 0");
- query(em, "review_score > 2");
- query(em, "review_score > 3");
- query(em, "review_score > 4");
- query(em, "review_score > 5");
- }
- }
-
- public static void query( EntityManager em, String query ) throws Exception {
- Query q = Query.fromQL(query) ;
- Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
- log.info("size = {} returned from query {}",results.size(), q.getQl() );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
new file mode 100644
index 0000000..a818199
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.NumberUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class Usegrid1PerformanceTest {
+ private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceTest.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityManagerFactory emf;
+
+
+ public Usegrid1PerformanceTest() throws Throwable {
+ emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
+ }
+
+
+ public EntityManagerFactory getEmf() {
+ return emf;
+ }
+
+
+ @Test
+ public void loadAndReadData() throws Exception {
+
+ log.info("Start Data Load");
+ List<UUID> apps = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( apps );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( apps );
+
+ }
+
+
+ private List<UUID> loadData() throws Exception {
+
+ long time = new Date().getTime();
+
+ List<UUID> apps = new ArrayList<UUID>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ UUID appId = getEmf().createApplication(
+ "testorg-" + RandomStringUtils.randomAlphanumeric(6),
+ "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+ apps.add( appId );
+
+ Thread t = new Thread( new DataLoader( appId ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return apps;
+ }
+
+
+ private void readData( List<UUID> apps ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( UUID app : apps ) {
+
+ Thread t = new Thread( new DataReader( app ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+
+ private class DataReader implements Runnable {
+ UUID app;
+
+ public DataReader( UUID app ) {
+ this.app = app;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ UUID appId = app;
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ return;
+ }
+
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ log.info("Read {} reviews in {}", count, appId );
+ return;
+ }
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {}", count, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ UUID app;
+
+ public DataLoader( UUID scope ) {
+ this.app = scope;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Map<String, Object> currentEntityMap = new HashMap<String, Object>();
+
+ UUID appId = app;
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ Entity entity = em.create("review", currentEntityMap );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", entity.getUuid());
+ log.info("---");
+ }
+
+ // create the next entity
+ currentEntityMap = new HashMap<String, Object>();
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {}", count, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ currentEntityMap.put( name, Double.parseDouble(value));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ currentEntityMap.put( name, Long.parseLong(value));
+
+ } else {
+ currentEntityMap.put( name, value.toString() );
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+ }
+ }
+
+
+ public void runSelectedQueries( List<UUID> apps ) throws Exception {
+
+ for ( UUID app : apps ) {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ query(em, "product_productid = 'B006K2ZZ7K'") ;
+ query(em, "review_profilename = 'Twoapennything'") ;
+ query(em, "review_profilename contains 'Natalia'") ;
+ query(em, "review_profilename contains 'Patrick'") ;
+ query(em, "review_time = 1342051200") ;
+ query(em, "review_time > 1342051200") ;
+ query(em, "review_score > 0");
+ query(em, "review_score > 2");
+ query(em, "review_score > 3");
+ query(em, "review_score > 4");
+ query(em, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityManager em, String query ) throws Exception {
+ Query q = Query.fromQL(query) ;
+ Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest2/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/pom.xml b/stack/corepersistence/perftest2/pom.xml
index be1a457..34ef45f 100644
--- a/stack/corepersistence/perftest2/pom.xml
+++ b/stack/corepersistence/perftest2/pom.xml
@@ -59,27 +59,6 @@
<extensions>true</extensions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
- <overWriteReleases>false</overWriteReleases>
- <overWriteSnapshots>false</overWriteSnapshots>
- <overWriteIfNewer>true</overWriteIfNewer>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
</plugins>
</build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
deleted file mode 100644
index 8536fca..0000000
--- a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence;
-
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.netflix.config.ConfigurationManager;
-import java.io.BufferedReader;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.index.EntityCollectionIndex;
-import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory;
-import org.apache.usergrid.persistence.index.guice.TestIndexModule;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.DoubleField;
-import org.apache.usergrid.persistence.model.field.LongField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.query.Query;
-import org.apache.usergrid.persistence.query.Results;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * TODO: make configurable, add CHOP markup.
- */
-public class CorePerformanceIT {
- private static final Logger log = LoggerFactory.getLogger(CorePerformanceIT.class);
-
- // max entities we will write and read
- static int maxEntities = Integer.MAX_VALUE;
-
- // each app will get all data
- static int orgCount = 2;
- static int appCount = 5 ;
-
- // number of threads = orgCount x appCount
-
- // total number of records = orgCount x appCount x numRecords
-
- private final EntityCollectionManagerFactory ecmf;
- private final EntityCollectionIndexFactory ecif;
-
- public CorePerformanceIT() {
- Injector injector = Guice.createInjector( new TestIndexModule() );
- ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
- ecif = injector.getInstance( EntityCollectionIndexFactory.class );
- }
-
- @Ignore
- @Test
- public void loadAndReadData() throws IOException, InterruptedException {
-
- ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
-
- // only on first run
- //MigrationManager m = injector.getInstance( MigrationManager.class )
- //m.migrate()
-
- log.info("Start Data Load");
- List<CollectionScope> scopes = loadData();
- log.info("Finish Data Load");
-
- log.info("Start Data Read");
- readData( scopes );
- log.info("Finish Data Read");
-
- runSelectedQueries( scopes );
-
- }
-
-
- private List<CollectionScope> loadData() throws InterruptedException {
-
- long time = new Date().getTime();
-
- List<CollectionScope> scopes = new ArrayList<CollectionScope>();
- List<Thread> threads = new ArrayList<Thread>();
-
- for ( int i=0; i<orgCount; i++ ) {
-
- String orgName = "org-" + i + "-" + time;
- final Id orgId = new SimpleId(orgName);
-
- for ( int j=0; j<appCount; j++ ) {
-
- String appName = "app-" + j + "-" + time;
- final Id appId = new SimpleId(appName);
-
- CollectionScope scope = new CollectionScopeImpl( orgId, appId, "reviews" );
- scopes.add( scope );
-
- Thread t = new Thread( new DataLoader( scope ));
- t.start();
- threads.add(t);
- }
- }
-
- // wait for indexing to end
- for ( Thread t : threads ) {
- t.join();
- }
-
- return scopes;
- }
-
-
- private void readData( List<CollectionScope> scopes ) throws InterruptedException {
-
- List<Thread> threads = new ArrayList<Thread>();
- for ( CollectionScope scope : scopes ) {
-
- Thread t = new Thread( new DataReader( scope ));
- t.start();
- threads.add(t);
- }
-
- // wait for reading to end
- for ( Thread t : threads ) {
- t.join();
- }
- }
-
- /**
- * @return the ecmf
- */
- public EntityCollectionManagerFactory getEcmf() {
- return ecmf;
- }
-
- /**
- * @return the ecif
- */
- public EntityCollectionIndexFactory getEcif() {
- return ecif;
- }
-
-
- private class DataReader implements Runnable {
- CollectionScope scope;
-
- public DataReader( CollectionScope scope ) {
- this.scope = scope;
- }
-
- public void run() {
-
- Id orgId = scope.getOrganization();
- Id appId = scope.getOwner();
-
- EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
- EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
-
- Query query = Query.fromQL( "review_score > 0"); // get all reviews;
- query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
-
- Results results = eci.execute( query );
- results.getEntities(); // cause retrieval from Cassandra
- int count = results.size();
-
- while ( results.hasCursor() && count < maxEntities ) {
- query.setCursor( results.getCursor() ) ;
- results = eci.execute( query );
- results.getEntities(); // cause retrieval from Cassanda;
- count += results.size();
-
- log.info("Read {} reviews in {} / {} ", count, orgId, appId );
- }
- }
- }
-
-
- private class DataLoader implements Runnable {
- CollectionScope scope;
-
- public DataLoader( CollectionScope scope ) {
- this.scope = scope;
- }
-
- public void run() {
-
- EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
- EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
-
- BufferedReader br;
- try {
- InputStreamReader isr = new InputStreamReader(
- getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
- br = new BufferedReader(isr);
- } catch (Exception ex) {
- throw new RuntimeException("Error opening file", ex);
- }
- String s = null;
-
- // create the first entry
- Entity current = new Entity(
- new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
-
- Id orgId = scope.getOrganization();
- Id appId = scope.getOwner();
-
- int count = 0;
- try {
- while ( (s = br.readLine()) != null && count < maxEntities ) {
-
- try {
-
- if ( s.trim().equals("")) { // then we are at end of a record
-
- // write and index current entity
- ecm.write( current ).toBlockingObservable().last();
- eci.index( current );
-
- if ( maxEntities < 20 ) {
- log.info("Index written for {}", current.getId());
- log.info("---");
- }
-
- // create the next entity
- current = new Entity(
- new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
-
- count++;
- if (count % 100000 == 0) {
- log.info("Indexed {} reviews in {} / {} ", count, orgId, appId );
- }
- continue;
- }
-
- // process a field
- String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
- String value = s.substring( s.indexOf(":") + 1 ).trim();
-
- if ( maxEntities < 20 ) {
- log.info("Indexing {} = {}", name, value);
- }
-
- if ( NumberUtils.isNumber(value) && value.contains(".")) {
- current.setField( new DoubleField( name, Double.parseDouble(value)));
-
- } else if ( NumberUtils.isNumber(value) ) {
- current.setField( new LongField( name, Long.parseLong(value)));
-
- } else {
- current.setField( new StringField( name, value.toString() ));
- }
-
- } catch ( Exception e ) {
- log.info("Error on line " + count);
- }
- }
-
- } catch (IOException ex) {
- throw new RuntimeException("Error reading file", ex);
- }
-
- eci.refresh();
- }
- }
-
-
- public void runSelectedQueries( List<CollectionScope> scopes ) {
-
- for ( CollectionScope scope : scopes ) {
-
- EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
- EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
-
- // TODO: come up with more and more complex queries for CorePerformanceIT
-
- query(eci, "product_productid = 'B006K2ZZ7K'") ;
- query(eci, "review_profilename = 'Twoapennything'") ;
- query(eci, "review_profilename contains 'Natalia'") ;
- query(eci, "review_profilename contains 'Patrick'") ;
- query(eci, "review_time = 1342051200") ;
- query(eci, "review_time > 1342051200") ;
- query(eci, "review_score > 0");
- query(eci, "review_score > 2");
- query(eci, "review_score > 3");
- query(eci, "review_score > 4");
- query(eci, "review_score > 5");
- }
- }
-
- public static void query( EntityCollectionIndex eci, String query ) {;
- Query q = Query.fromQL(query) ;
- Results results = eci.execute( q );
- log.info("size = {} returned from query {}",results.size(), q.getQl() );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceTest.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceTest.java
new file mode 100644
index 0000000..1e69167
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceTest.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.netflix.config.ConfigurationManager;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.index.EntityCollectionIndex;
+import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.query.Query;
+import org.apache.usergrid.persistence.query.Results;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class CorePerformanceTest {
+ private static final Logger log = LoggerFactory.getLogger(CorePerformanceTest.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityCollectionManagerFactory ecmf;
+ private final EntityCollectionIndexFactory ecif;
+
+ public CorePerformanceTest() {
+ Injector injector = Guice.createInjector( new TestIndexModule() );
+ ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
+ ecif = injector.getInstance( EntityCollectionIndexFactory.class );
+ }
+
+
+ @Test
+ public void loadAndReadData() throws IOException, InterruptedException {
+
+ ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
+
+ // only on first run
+ //MigrationManager m = injector.getInstance( MigrationManager.class )
+ //m.migrate()
+
+ log.info("Start Data Load");
+ List<CollectionScope> scopes = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( scopes );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( scopes );
+
+ }
+
+
+ private List<CollectionScope> loadData() throws InterruptedException {
+
+ long time = new Date().getTime();
+
+ List<CollectionScope> scopes = new ArrayList<CollectionScope>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ String orgName = "org-" + i + "-" + time;
+ final Id orgId = new SimpleId(orgName);
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ String appName = "app-" + j + "-" + time;
+ final Id appId = new SimpleId(appName);
+
+ CollectionScope scope = new CollectionScopeImpl( orgId, appId, "reviews" );
+ scopes.add( scope );
+
+ Thread t = new Thread( new DataLoader( scope ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return scopes;
+ }
+
+
+ private void readData( List<CollectionScope> scopes ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( CollectionScope scope : scopes ) {
+
+ Thread t = new Thread( new DataReader( scope ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+ /**
+ * @return the ecmf
+ */
+ public EntityCollectionManagerFactory getEcmf() {
+ return ecmf;
+ }
+
+ /**
+ * @return the ecif
+ */
+ public EntityCollectionIndexFactory getEcif() {
+ return ecif;
+ }
+
+
+ private class DataReader implements Runnable {
+ CollectionScope scope;
+
+ public DataReader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {} / {} ", count, orgId, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ CollectionScope scope;
+
+ public DataLoader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Entity current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ ecm.write( current ).toBlockingObservable().last();
+ eci.index( current );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", current.getId());
+ log.info("---");
+ }
+
+ // create the next entity
+ current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {} / {} ", count, orgId, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ current.setField( new DoubleField( name, Double.parseDouble(value)));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ current.setField( new LongField( name, Long.parseLong(value)));
+
+ } else {
+ current.setField( new StringField( name, value.toString() ));
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+
+ eci.refresh();
+ }
+ }
+
+
+ public void runSelectedQueries( List<CollectionScope> scopes ) {
+
+ for ( CollectionScope scope : scopes ) {
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ // TODO: come up with more and more complex queries for CorePerformanceIT
+
+ query(eci, "product_productid = 'B006K2ZZ7K'") ;
+ query(eci, "review_profilename = 'Twoapennything'") ;
+ query(eci, "review_profilename contains 'Natalia'") ;
+ query(eci, "review_profilename contains 'Patrick'") ;
+ query(eci, "review_time = 1342051200") ;
+ query(eci, "review_time > 1342051200") ;
+ query(eci, "review_score > 0");
+ query(eci, "review_score > 2");
+ query(eci, "review_score > 3");
+ query(eci, "review_score > 4");
+ query(eci, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityCollectionIndex eci, String query ) {;
+ Query q = Query.fromQL(query) ;
+ Results results = eci.execute( q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}
[18/18] git commit: Merge branch 'two-dot-o' into hystrix-integration
Posted by sn...@apache.org.
Merge branch 'two-dot-o' into hystrix-integration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b1d12a17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b1d12a17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b1d12a17
Branch: refs/heads/hystrix-integration
Commit: b1d12a17176dbcaa518cc41430322bad11550557
Parents: 58f7f43 fc10597
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 24 08:16:49 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 24 08:16:49 2014 -0700
----------------------------------------------------------------------
stack/awscluster/README.md | 2 +-
.../src/main/groovy/wait_for_instances.groovy | 2 +-
stack/corepersistence/perftest1/pom.xml | 9 ++---
.../persistence/Usergrid1PerformanceTest.java | 36 +++++++++++---------
4 files changed, 24 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
[02/18] git commit: Fixes to CorePersistenceIT and index impl.
Posted by sn...@apache.org.
Fixes to CorePersistenceIT and index impl.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6b7ffbeb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6b7ffbeb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6b7ffbeb
Branch: refs/heads/hystrix-integration
Commit: 6b7ffbeb59f4b04657c874b284349675e52c4dba
Parents: b3b46e4
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Mar 20 13:19:46 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Mar 20 13:19:46 2014 -0400
----------------------------------------------------------------------
.../index/impl/EsEntityCollectionIndex.java | 13 ++++++++-----
.../index/impl/CorePerformanceIT.java | 20 ++++++++++----------
2 files changed, 18 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b7ffbeb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java
index 4d750a7..8e16d88 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityCollectionIndex.java
@@ -49,7 +49,6 @@ import org.apache.usergrid.persistence.model.field.SetField;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.query.Query;
import org.apache.usergrid.persistence.query.Results;
-import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.types.TypesExistsRequest;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
@@ -119,13 +118,17 @@ public class EsEntityCollectionIndex implements EntityCollectionIndex {
this.refresh = config.isForcedRefresh();
this.cursorTimeout = config.getQueryCursorTimeout();
- // if new index then create it
AdminClient admin = client.admin();
- if (!admin.indices().exists(
- new IndicesExistsRequest(indexName)).actionGet().isExists()) {
-
+ try {
admin.indices().prepareCreate(indexName).execute().actionGet();
log.debug("Created new index: " + indexName);
+
+ } catch (Exception e) {
+ if ( log.isDebugEnabled() ) {
+ log.debug("Exception creating index, already exists?", e);
+ } else {
+ log.info(e.getMessage());
+ }
}
// if new type then create mapping
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b7ffbeb/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
index b368b92..0008304 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
@@ -107,20 +107,17 @@ public class CorePerformanceIT {
for ( int i=0; i<orgCount; i++ ) {
- String orgName = "org-${i}-${time}";
+ String orgName = "org-" + i + "-" + time;
final Id orgId = new SimpleId(orgName);
for ( int j=0; j<appCount; j++ ) {
- String appName = "app-${j}-${time}";
+ String appName = "app-" + j + "-" + time;
final Id appId = new SimpleId(appName);
CollectionScope scope = new CollectionScopeImpl( orgId, appId, "reviews" );
scopes.add( scope );
-// def scopeFile = new File("/home/ubuntu/scopes.txt")
-// scopeFile.append("Created ${orgId}, ${appId}\n")
-
Thread t = new Thread( new DataLoader( scope ));
t.start();
threads.add(t);
@@ -181,7 +178,7 @@ public class CorePerformanceIT {
results.getEntities(); // cause retrieval from Cassanda;
count += results.size();
- log.info("Read ${count} reviews in ${orgId} ${appId}");
+ log.info("Read {} reviews in {} / {} ", count, orgId, appId );
}
}
}
@@ -212,6 +209,9 @@ public class CorePerformanceIT {
Entity current = new Entity(
new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
int count = 0;
try {
while ( (s = br.readLine()) != null && count < maxEntities ) {
@@ -225,7 +225,7 @@ public class CorePerformanceIT {
eci.index( current );
if ( maxEntities < 20 ) {
- log.info("Index written for ${current.getId()}");
+ log.info("Index written for {}", current.getId());
log.info("---");
}
@@ -235,7 +235,7 @@ public class CorePerformanceIT {
count++;
if (count % 100000 == 0) {
- log.info("Indexed ${count} reviews in ${orgId} ${appId}");
+ log.info("Indexed {} reviews in {} / {} ", count, orgId, appId );
}
continue;
}
@@ -245,7 +245,7 @@ public class CorePerformanceIT {
String value = s.substring( s.indexOf(":") + 1 ).trim();
if ( maxEntities < 20 ) {
- log.info("Indexing ${name} = ${value}");
+ log.info("Indexing {} = {}", name, value);
}
if ( NumberUtils.isNumber(value) && value.contains(".")) {
@@ -298,7 +298,7 @@ public class CorePerformanceIT {
public static void query( EntityCollectionIndex eci, String query ) {;
Query q = Query.fromQL(query) ;
Results results = eci.execute( q );
- log.info("${q.getQl()}: ${results.getIds().size()}");
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
}
}
[14/18] git commit: Fixed typo in readme
Posted by sn...@apache.org.
Fixed typo in readme
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a4dd97aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a4dd97aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a4dd97aa
Branch: refs/heads/hystrix-integration
Commit: a4dd97aae7b368107bd50071072a8001fa83e017
Parents: a705630
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 24 08:16:02 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 24 08:16:02 2014 -0700
----------------------------------------------------------------------
stack/awscluster/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a4dd97aa/stack/awscluster/README.md
----------------------------------------------------------------------
diff --git a/stack/awscluster/README.md b/stack/awscluster/README.md
index b7558b6..476601a 100644
--- a/stack/awscluster/README.md
+++ b/stack/awscluster/README.md
@@ -32,7 +32,7 @@ be named `jdk-7u45-linux-x64.gz`.
* __Create an aws.properties file__ with your AWS credentials in the same directory as this
README file. The file is git-ignored so you don't have to worry about accidentally committing it.
-* __Deploy this the AWS Cluster assembly__ by running the Maven command `mvn depoy` in the same
+* __Deploy this the AWS Cluster assembly__ by running the Maven command `mvn deploy` in the same
directory as this README file.
[17/18] git commit: Merge branch 'two-dot-o' of
github.com:usergrid/usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of github.com:usergrid/usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fc105970
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fc105970
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fc105970
Branch: refs/heads/hystrix-integration
Commit: fc105970fa93f486b24cc1fbfc0df85349d96f97
Parents: a4dd97a c4f9d0f
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 24 08:16:24 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 24 08:16:24 2014 -0700
----------------------------------------------------------------------
stack/corepersistence/perftest1/pom.xml | 9 ++---
.../persistence/Usergrid1PerformanceTest.java | 36 +++++++++++---------
2 files changed, 22 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
[15/18] git commit: Fixed typo in readme
Posted by sn...@apache.org.
Fixed typo in readme
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a4dd97aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a4dd97aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a4dd97aa
Branch: refs/heads/two-dot-o
Commit: a4dd97aae7b368107bd50071072a8001fa83e017
Parents: a705630
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 24 08:16:02 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 24 08:16:02 2014 -0700
----------------------------------------------------------------------
stack/awscluster/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a4dd97aa/stack/awscluster/README.md
----------------------------------------------------------------------
diff --git a/stack/awscluster/README.md b/stack/awscluster/README.md
index b7558b6..476601a 100644
--- a/stack/awscluster/README.md
+++ b/stack/awscluster/README.md
@@ -32,7 +32,7 @@ be named `jdk-7u45-linux-x64.gz`.
* __Create an aws.properties file__ with your AWS credentials in the same directory as this
README file. The file is git-ignored so you don't have to worry about accidentally committing it.
-* __Deploy this the AWS Cluster assembly__ by running the Maven command `mvn depoy` in the same
+* __Deploy this the AWS Cluster assembly__ by running the Maven command `mvn deploy` in the same
directory as this README file.
[05/18] git commit: Write,
index and read performance test modules for new Core Persistence API
and old Entity Manager API.
Posted by sn...@apache.org.
Write, index and read performance test modules for new Core Persistence API and old Entity Manager API.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d2cb960f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d2cb960f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d2cb960f
Branch: refs/heads/hystrix-integration
Commit: d2cb960f40de016f1c8bbaa3a54e9fbc295e7e9e
Parents: e2f6fbf
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Mar 21 09:55:35 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Mar 21 09:55:35 2014 -0400
----------------------------------------------------------------------
.gitignore | 6 +
stack/corepersistence/perftest1/pom.xml | 97 ++++++
.../usergrid/persistence/UsergridBootstrap.java | 303 +++++++++++++++++
.../src/main/resources/log4j.properties | 44 +++
.../src/main/resources/project.properties | 1 +
.../main/resources/usergrid-test-context.xml | 39 +++
.../persistence/Usegrid1PerformanceIT.java | 304 ++++++++++++++++++
.../persistence/UsergridBootstrapTest.java | 56 ++++
stack/corepersistence/perftest2/pom.xml | 85 +++++
.../main/groovy/perftest2/CreateEntity.groovy | 100 ++++++
.../usergrid/persistence/CorePerformanceIT.java | 321 +++++++++++++++++++
.../index/guice/TestIndexModule.java | 30 ++
12 files changed, 1386 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 38c4045..c9f4468 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,9 @@ stack/corepersistence/nbactions.xml
stack/corepersistence/queryindex/nbactions.xml
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
+/stack/corepersistence/perftest2/src/main/resources/finefoods.txt
+/stack/corepersistence/perftest1/src/main/resources/finefoods.txt
+/stack/corepersistence/perftest2/nbactions.xml
+/stack/corepersistence/perftest1/nbactions.xml
+/stack/corepersistence/perftest1/src/main/resources/usergrid.properties
+/stack/corepersistence/perftest2/src/main/resources/usergrid.properties
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/pom.xml b/stack/corepersistence/perftest1/pom.xml
new file mode 100644
index 0000000..abee483
--- /dev/null
+++ b/stack/corepersistence/perftest1/pom.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>perftest1</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.compiler.source>1.7</maven.compiler.source>
+ <maven.compiler.target>1.7</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-core</artifactId>
+ <version>0.0.29-SNAPSHOT</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-test-utils</artifactId>
+ <version>0.0.29-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <artifactId>groovy-all</artifactId>
+ <groupId>org.codehaus.groovy</groupId>
+ <version>2.2.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-tools</artifactId>
+ <version>0.0.29-SNAPSHOT</version>
+ <scope>test</scope>
+ <type>jar</type>
+ </dependency>
+ </dependencies>
+
+ <name>perftest1</name>
+
+ <build>
+ <plugins>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.3.2</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>2.6.0-01</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <groupId>org.codehaus.groovy</groupId>
+ <version>2.6.0-01</version>
+ <extensions>true</extensions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java b/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java
new file mode 100644
index 0000000..67f4af4
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/java/org/apache/usergrid/persistence/UsergridBootstrap.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.usergrid.cassandra.SchemaManager;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.commons.lang.StringUtils;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_APPLICATION;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_ORGANIZATION;
+
+
+/**
+ * Does the Spring magic to get bootstrap a Usergrid CassandraService, Entity Manager Factory and
+ * entity manager. Based on code from CassandraResource in the test-utils module.
+ * <p/>
+ * Note that for this resource to work properly, a project.properties file must be placed in the
+ * src/test/resources directory with the following stanza in the project pom's build section:
+ * <p/>
+ * <testResources> <testResource> <directory>src/test/resources</directory> <filtering>true</filtering> <includes>
+ * <include>**\/*.properties</include> <include>**\/*.xml</include> </includes> </testResource> </testResources>
+ * <p/>
+ * The following property expansion macro should be placed in this project.properties file:
+ * <p/>
+ * target.directory=${pom.build.directory}
+ */
+public class UsergridBootstrap { // is that a great classname or what!
+
+ public static final Logger LOG = LoggerFactory.getLogger( UsergridBootstrap.class );
+
+ public static final String PROPERTIES_FILE = "project.properties";
+
+ public static final String TARGET_DIRECTORY_KEY = "target.directory";
+
+ private static final Object lock = new Object();
+
+ private final File tempDir;
+ private final String schemaManagerName;
+
+ private boolean initialized = false;
+
+ private ConfigurableApplicationContext applicationContext;
+ private SchemaManager schemaManager;
+
+ private static UsergridBootstrap instance;
+ private Thread shutdown;
+
+
+ /**
+ * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the
+ * specified SchemaManager for Cassandra.
+ */
+ UsergridBootstrap( String schemaManagerName ) throws Throwable {
+ LOG.info( "Creating CassandraResource using {} for the ClassLoader.",
+ Thread.currentThread().getContextClassLoader() );
+
+ this.schemaManagerName = schemaManagerName;
+ try {
+ this.tempDir = getTempDirectory();
+ }
+ catch ( Exception e ) {
+ LOG.error( "Failed to create temporary directory for Cassandra instance.", e );
+ throw new RuntimeException( e );
+ }
+
+ before();
+ }
+
+
+ /**
+ * Gets the temporary directory created for this CassandraResource.
+ *
+ * @return the temporary directory
+ */
+ @SuppressWarnings("UnusedDeclaration")
+ public File getTemporaryDirectory() {
+ return tempDir;
+ }
+
+
+ /**
+ * Gets a bean from the application context.
+ *
+ * @param requiredType the type of the bean
+ * @param <T> the type of the bean
+ *
+ * @return the bean
+ */
+ public <T> T getBean( String name, Class<T> requiredType ) {
+ protect();
+ return applicationContext.getBean( name, requiredType );
+ }
+
+
+ /**
+ * Gets a bean from the application context.
+ *
+ * @param requiredType the type of the bean
+ * @param <T> the type of the bean
+ *
+ * @return the bean
+ */
+ public <T> T getBean( Class<T> requiredType ) {
+ protect();
+ return applicationContext.getBean( requiredType );
+ }
+
+
+ /**
+ * Gets whether this resource is ready to use.
+ *
+ * @return true if ready to use, false otherwise
+ */
+ public boolean isReady() {
+ return initialized;
+ }
+
+ /**
+ * Protects against IDE or command line runs of a unit test which when starting the test
+ * outside of the Suite will not start the resource. This makes sure the resource is
+ * automatically started on a usage attempt.
+ */
+ private void protect() {
+ if ( !isReady() ) {
+ try {
+ before();
+ }
+ catch ( Throwable t ) {
+ LOG.error( "Failed to start up Cassandra.", t );
+ throw new RuntimeException( t );
+ }
+ }
+ }
+
+
+ /**
+ * before TestSuite or test Class execution.
+ *
+ * @throws Throwable if we cannot start up Cassandra
+ */
+ protected void before() throws Throwable {
+
+ String[] locations = { "usergrid-test-context.xml" };
+ applicationContext = new ClassPathXmlApplicationContext( locations );
+
+ EntityManagerFactory emf = applicationContext.getBean( EntityManagerFactory.class );
+
+ String appName = StringUtils.lowerCase( DEFAULT_APPLICATION.contains( "/" )
+ ? DEFAULT_APPLICATION : DEFAULT_ORGANIZATION + "/" + DEFAULT_APPLICATION );
+
+ if ( emf.lookupApplication(appName) == null ) {
+ loadSchemaManager( schemaManagerName );
+ }
+
+ initialized = true;
+ LOG.info( "External Cassandra resource at is ready!");
+ lock.notifyAll();
+ }
+
+
+ /** Stops Cassandra after a TestSuite or test Class executes. */
+ protected synchronized void after() {
+
+ shutdown = new Thread() {
+ @Override
+ public void run() {
+ try {
+ Thread.currentThread().sleep( 100L );
+ }
+ catch ( InterruptedException e ) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+
+ if ( schemaManager != null ) {
+ LOG.info( "Destroying schemaManager..." );
+ try {
+ schemaManager.destroy();
+ }
+ catch ( Exception e ) {
+ LOG.error( "Ignoring failures while dropping keyspaces: {}", e.getMessage() );
+ }
+
+ LOG.info( "SchemaManager destroyed..." );
+ }
+
+ applicationContext.stop();
+ LOG.info( "ApplicationContext stopped..." );
+
+ }
+ };
+
+ shutdown.start();
+ }
+
+
+ /**
+ * Loads the specified {@link SchemaManager} or the default manager if the manager
+ * name that is provided is null.
+ *
+ * @param schemaManagerName the name of the SchemaManager to load, or null
+ */
+ private void loadSchemaManager( String schemaManagerName ) {
+ if ( !applicationContext.isActive() ) {
+ LOG.info( "Restarting context..." );
+ applicationContext.refresh();
+ }
+
+ if ( schemaManagerName != null ) {
+ LOG.info( "Looking up SchemaManager impl: {}", schemaManagerName );
+ this.schemaManager = applicationContext.getBean( schemaManagerName, SchemaManager.class );
+ }
+ else {
+ LOG.info( "The SchemaManager is not specified - using the default SchemaManager impl" );
+ this.schemaManager = applicationContext.getBean( SchemaManager.class );
+ }
+
+ schemaManager.create();
+ schemaManager.populateBaseData();
+ }
+
+
+ /**
+ * Creates a new instance
+ *
+ * @param schemaManagerName the name of the schemaManager to use
+ *
+ * @return a new CassandraResource with possibly non-default ports
+ */
+ public static UsergridBootstrap newInstance( String schemaManagerName ) throws Throwable {
+ // Uncomment to test for Surefire Failures
+ // System.setSecurityManager( new NoExitSecurityManager( System.getSecurityManager() ) );
+
+ synchronized ( lock ) {
+ if ( instance != null ) {
+ return instance;
+ }
+
+ instance = new UsergridBootstrap( schemaManagerName );
+ LOG.info("Created a new instance of CassandraResource: {}", instance);
+ return instance;
+ }
+ }
+
+
+ public static UsergridBootstrap newInstance() throws Throwable {
+ return newInstance( null );
+ }
+
+
+ /**
+ * Uses a project.properties file that Maven does substitution on to to replace the value of
+ * a property with the path to the Maven build directory (a.k.a. target). It then uses this
+ * path to generate a random String which it uses to append a path component to so a unique
+ * directory is selected. If already present it's deleted, then the directory is created.
+ *
+ * @return a unique temporary directory
+ *
+ * @throws IOException if we cannot access the properties file
+ */
+ public static File getTempDirectory() throws IOException {
+ File tmpdir;
+ Properties props = new Properties();
+ props.load( ClassLoader.getSystemResourceAsStream( PROPERTIES_FILE ) );
+ File basedir = new File( ( String ) props.get( TARGET_DIRECTORY_KEY ) );
+ String comp = RandomStringUtils.randomAlphanumeric( 7 );
+ tmpdir = new File( basedir, comp );
+
+ if ( tmpdir.exists() ) {
+ LOG.info( "Deleting directory: {}", tmpdir );
+ FileUtils.forceDelete( tmpdir );
+ }
+ else {
+ LOG.info( "Creating temporary directory: {}", tmpdir );
+ FileUtils.forceMkdir( tmpdir );
+ }
+
+ return tmpdir;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/resources/log4j.properties b/stack/corepersistence/perftest1/src/main/resources/log4j.properties
new file mode 100644
index 0000000..fab8d7b
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/resources/log4j.properties
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# for production, you should probably set the root to INFO
+# and the pattern to %c instead of %l. (%l is slower.)
+
+# output messages into a rolling log file as well as stdout
+log4j.rootLogger=INFO,stdout
+
+# stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+#log4j.appender.stdout.layout=org.apache.log4j.SimpleLayout
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
+
+log4j.category.org.apache=ERROR, stdout
+
+log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.DaoUtils=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.ConnectionRefImpl=WARN, stdout
+log4j.logger.me.prettyprint.cassandra.hector.TimingLogger=WARN, stdout
+log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
+log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
+#log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
+
+
+#log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/resources/project.properties b/stack/corepersistence/perftest1/src/main/resources/project.properties
new file mode 100644
index 0000000..c0bf364
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/resources/project.properties
@@ -0,0 +1 @@
+target.directory=${project.build.directory}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml b/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml
new file mode 100644
index 0000000..9bdabe6
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/main/resources/usergrid-test-context.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:util="http://www.springframework.org/schema/util"
+ xmlns:context="http://www.springframework.org/schema/context" xmlns:p="http://www.springframework.org/schema/p"
+ xmlns:hz="http://www.hazelcast.com/schema/config" xmlns:aop="http://www.springframework.org/schema/aop"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
+ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util-3.1.xsd
+ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd">
+
+ <!-- configure our test properties -->
+ <bean id="properties"
+ class="org.springframework.beans.factory.config.PropertiesFactoryBean">
+ <property name="singleton" value="true" />
+ <property name="ignoreResourceNotFound" value="true" />
+ <property name="locations">
+ <list>
+ <value>classpath:/usergrid-default.properties</value>
+ <value>classpath:/usergrid.properties</value>
+ <value>classpath:/usergrid-scheduler-test.properties</value>
+ </list>
+ </property>
+ </bean>
+
+ <import resource="classpath:/usergrid-core-context.xml"/>
+
+ <bean id="setup" class="org.apache.usergrid.persistence.cassandra.Setup">
+ <constructor-arg ref="entityManagerFactory"/>
+ <constructor-arg ref="cassandraService"/>
+ </bean>
+
+ <!-- Refer to a named schemaManager from the DataControl annotation thusly -->
+ <bean id="coreManager" class="org.apache.usergrid.persistence.CoreSchemaManager">
+ <constructor-arg ref="setup"/>
+ <constructor-arg ref="cassandraCluster"/>
+ </bean>
+</beans>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
new file mode 100644
index 0000000..af47a8c
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.NumberUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class Usegrid1PerformanceIT {
+ private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceIT.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityManagerFactory emf;
+
+
+ public Usegrid1PerformanceIT() throws Throwable {
+ emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
+ }
+
+
+ public EntityManagerFactory getEmf() {
+ return emf;
+ }
+
+
+ @Ignore
+ @Test
+ public void loadAndReadData() throws Exception {
+
+ log.info("Start Data Load");
+ List<UUID> apps = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( apps );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( apps );
+
+ }
+
+
+ private List<UUID> loadData() throws Exception {
+
+ long time = new Date().getTime();
+
+ List<UUID> apps = new ArrayList<UUID>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ UUID appId = getEmf().createApplication(
+ "testorg-" + RandomStringUtils.randomAlphanumeric(6),
+ "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+ apps.add( appId );
+
+ Thread t = new Thread( new DataLoader( appId ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return apps;
+ }
+
+
+ private void readData( List<UUID> apps ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( UUID app : apps ) {
+
+ Thread t = new Thread( new DataReader( app ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+
+ private class DataReader implements Runnable {
+ UUID app;
+
+ public DataReader( UUID app ) {
+ this.app = app;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ UUID appId = app;
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ return;
+ }
+
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ log.info("Read {} reviews in {}", count, appId );
+ return;
+ }
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {}", count, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ UUID app;
+
+ public DataLoader( UUID scope ) {
+ this.app = scope;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Map<String, Object> currentEntityMap = new HashMap<String, Object>();
+
+ UUID appId = app;
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ Entity entity = em.create("review", currentEntityMap );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", entity.getUuid());
+ log.info("---");
+ }
+
+ // create the next entity
+ currentEntityMap = new HashMap<String, Object>();
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {}", count, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ currentEntityMap.put( name, Double.parseDouble(value));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ currentEntityMap.put( name, Long.parseLong(value));
+
+ } else {
+ currentEntityMap.put( name, value.toString() );
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+ }
+ }
+
+
+ public void runSelectedQueries( List<UUID> apps ) throws Exception {
+
+ for ( UUID app : apps ) {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ query(em, "product_productid = 'B006K2ZZ7K'") ;
+ query(em, "review_profilename = 'Twoapennything'") ;
+ query(em, "review_profilename contains 'Natalia'") ;
+ query(em, "review_profilename contains 'Patrick'") ;
+ query(em, "review_time = 1342051200") ;
+ query(em, "review_time > 1342051200") ;
+ query(em, "review_score > 0");
+ query(em, "review_score > 2");
+ query(em, "review_score > 3");
+ query(em, "review_score > 4");
+ query(em, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityManager em, String query ) throws Exception {
+ Query q = Query.fromQL(query) ;
+ Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java
new file mode 100644
index 0000000..dfdac5d
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/UsergridBootstrapTest.java
@@ -0,0 +1,56 @@
+
+package org.apache.usergrid.persistence;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.RandomStringUtils;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class UsergridBootstrapTest {
+ private static final Logger log = LoggerFactory.getLogger(UsergridBootstrapTest.class);
+
+ @Ignore // just an example and not included in performance test
+ @Test
+ public void testBasicOperation() throws Throwable {
+
+ UsergridBootstrap sbf = UsergridBootstrap.newInstance();
+ EntityManagerFactory emf = sbf.getBean( EntityManagerFactory.class );
+
+ UUID appId = emf.createApplication(
+ "testorg-" + RandomStringUtils.randomAlphanumeric(6),
+ "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+ EntityManager em = emf.getEntityManager( appId );
+
+ Map<String, Object> map = new HashMap<String, Object>() {{
+ put("name", RandomStringUtils.randomAlphanumeric(6));
+ put("timestamp", new Date().getTime());
+ }};
+ Entity entity = em.create( "testentity", map );
+ String name = (String)entity.getProperty( "name" );
+
+ Entity got = em.get( entity.getUuid() );
+ assertNotNull(got);
+ String returnedName = (String)got.getProperty("name");
+ long timestamp = (long)got.getProperty("timestamp");
+ assertEquals( name, returnedName );
+ log.info(">>>> Got back name={} : time={}", returnedName, timestamp);
+
+ Query query = Query.fromQL("name = '" + name + "'");
+ Entity found = em.searchCollection( em.getApplication(), "testentities", query ).getEntity();
+ assertNotNull(found);
+ returnedName = (String)found.getProperty("name");
+ timestamp = (long)found.getProperty("timestamp");
+ assertEquals( name, returnedName );
+ log.info(">>>> Found name={} : time={}", returnedName, timestamp);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/pom.xml b/stack/corepersistence/perftest2/pom.xml
new file mode 100644
index 0000000..be1a457
--- /dev/null
+++ b/stack/corepersistence/perftest2/pom.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>perftest2</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <maven.compiler.source>1.7</maven.compiler.source>
+ <maven.compiler.target>1.7</maven.compiler.target>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>queryindex</artifactId>
+ <version>1.0-SNAPSHOT</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <artifactId>groovy-all</artifactId>
+ <groupId>org.codehaus.groovy</groupId>
+ <version>2.2.2</version>
+ </dependency>
+
+ </dependencies>
+
+ <name>perftest2</name>
+
+ <build>
+ <plugins>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.3.2</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>2.6.0-01</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <compilerId>groovy-eclipse-compiler</compilerId>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <groupId>org.codehaus.groovy</groupId>
+ <version>2.6.0-01</version>
+ <extensions>true</extensions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.8</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
+ <overWriteReleases>false</overWriteReleases>
+ <overWriteSnapshots>false</overWriteSnapshots>
+ <overWriteIfNewer>true</overWriteIfNewer>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy b/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy
new file mode 100644
index 0000000..d1a6477
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/main/groovy/perftest2/CreateEntity.groovy
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package perftest2;
+
+import com.google.inject.Injector
+import com.netflix.config.ConfigurationManager
+import org.apache.commons.lang3.RandomStringUtils
+import org.apache.usergrid.persistence.collection.CollectionScope
+import org.apache.usergrid.persistence.collection.EntityCollectionManager
+import com.google.inject.Guice
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl
+import org.apache.usergrid.persistence.collection.migration.MigrationManager
+import org.apache.usergrid.persistence.collection.util.EntityUtils
+import org.apache.usergrid.persistence.index.EntityCollectionIndex
+import org.apache.usergrid.persistence.index.guice.IndexModule
+import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory
+import org.apache.usergrid.persistence.model.entity.Entity
+import org.apache.usergrid.persistence.model.entity.Id
+import org.apache.usergrid.persistence.model.entity.SimpleId
+import org.apache.usergrid.persistence.model.field.LongField
+import org.apache.usergrid.persistence.model.field.StringField
+import org.apache.usergrid.persistence.model.util.UUIDGenerator
+import org.apache.usergrid.persistence.query.Query
+import org.apache.usergrid.persistence.query.Results
+
+
+/**
+ * Just an example and not included in performance test.
+ */
+public class CreateEntity {
+
+ public static void main( String[] args ) {
+
+ // setup core persistence
+ ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" )
+ Injector injector = Guice.createInjector( new IndexModule() )
+
+ // only on first run
+ //MigrationManager m = injector.getInstance( MigrationManager.class )
+ //m.migrate()
+
+ EntityCollectionManagerFactory ecmf =
+ injector.getInstance( EntityCollectionManagerFactory.class )
+ EntityCollectionIndexFactory ecif =
+ injector.getInstance( EntityCollectionIndexFactory.class )
+
+ // create scope
+ Id appId = new SimpleId("entitytest")
+ Id orgId = new SimpleId("usergrid")
+ CollectionScope scope = new CollectionScopeImpl( appId, orgId, "testentities" )
+
+ // entity manager and index interfaces
+ EntityCollectionManager ecm = ecmf.createCollectionManager( scope )
+ EntityCollectionIndex eci = ecif.createCollectionIndex( scope )
+
+ // create entity
+ Entity entity = new Entity(new SimpleId(UUIDGenerator.newTimeUUID(), "testentity"))
+ String name = RandomStringUtils.randomAlphanumeric(10)
+ entity.setField( new StringField("name", name))
+ entity.setField( new LongField("timestamp", new Date().getTime()))
+
+ // write and index entity
+ ecm.write( entity ).toBlockingObservable().last()
+ eci.index( entity )
+
+ eci.refresh()
+
+ ecm = ecmf.createCollectionManager( scope )
+ eci = ecif.createCollectionIndex( scope )
+
+ // get back entity
+ def got = ecm.load( entity.getId() ).toBlockingObservable().last()
+
+ def returnedName = got.getField("name").getValue()
+ def timestamp = got.getField("timestamp").getValue()
+ println ">>>> Got back name=${returnedName} : time=${timestamp}"
+
+ // search for entity
+ def found = eci.execute( Query.fromQL( "name = '${name}'".toString() ) ).getEntities().get(0)
+
+ returnedName = found.getField("name").getValue()
+ timestamp = found.getField("timestamp").getValue()
+ println ">>>> Found name=${returnedName} : time=${timestamp}"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
new file mode 100644
index 0000000..8536fca
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.netflix.config.ConfigurationManager;
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.index.EntityCollectionIndex;
+import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.query.Query;
+import org.apache.usergrid.persistence.query.Results;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class CorePerformanceIT {
+ private static final Logger log = LoggerFactory.getLogger(CorePerformanceIT.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityCollectionManagerFactory ecmf;
+ private final EntityCollectionIndexFactory ecif;
+
+ public CorePerformanceIT() {
+ Injector injector = Guice.createInjector( new TestIndexModule() );
+ ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
+ ecif = injector.getInstance( EntityCollectionIndexFactory.class );
+ }
+
+ @Ignore
+ @Test
+ public void loadAndReadData() throws IOException, InterruptedException {
+
+ ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
+
+ // only on first run
+ //MigrationManager m = injector.getInstance( MigrationManager.class )
+ //m.migrate()
+
+ log.info("Start Data Load");
+ List<CollectionScope> scopes = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( scopes );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( scopes );
+
+ }
+
+
+ private List<CollectionScope> loadData() throws InterruptedException {
+
+ long time = new Date().getTime();
+
+ List<CollectionScope> scopes = new ArrayList<CollectionScope>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ String orgName = "org-" + i + "-" + time;
+ final Id orgId = new SimpleId(orgName);
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ String appName = "app-" + j + "-" + time;
+ final Id appId = new SimpleId(appName);
+
+ CollectionScope scope = new CollectionScopeImpl( orgId, appId, "reviews" );
+ scopes.add( scope );
+
+ Thread t = new Thread( new DataLoader( scope ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return scopes;
+ }
+
+
+ private void readData( List<CollectionScope> scopes ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( CollectionScope scope : scopes ) {
+
+ Thread t = new Thread( new DataReader( scope ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+ /**
+ * @return the ecmf
+ */
+ public EntityCollectionManagerFactory getEcmf() {
+ return ecmf;
+ }
+
+ /**
+ * @return the ecif
+ */
+ public EntityCollectionIndexFactory getEcif() {
+ return ecif;
+ }
+
+
+ private class DataReader implements Runnable {
+ CollectionScope scope;
+
+ public DataReader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {} / {} ", count, orgId, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ CollectionScope scope;
+
+ public DataLoader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Entity current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ ecm.write( current ).toBlockingObservable().last();
+ eci.index( current );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", current.getId());
+ log.info("---");
+ }
+
+ // create the next entity
+ current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {} / {} ", count, orgId, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ current.setField( new DoubleField( name, Double.parseDouble(value)));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ current.setField( new LongField( name, Long.parseLong(value)));
+
+ } else {
+ current.setField( new StringField( name, value.toString() ));
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+
+ eci.refresh();
+ }
+ }
+
+
+ public void runSelectedQueries( List<CollectionScope> scopes ) {
+
+ for ( CollectionScope scope : scopes ) {
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ // TODO: come up with more and more complex queries for CorePerformanceIT
+
+ query(eci, "product_productid = 'B006K2ZZ7K'") ;
+ query(eci, "review_profilename = 'Twoapennything'") ;
+ query(eci, "review_profilename contains 'Natalia'") ;
+ query(eci, "review_profilename contains 'Patrick'") ;
+ query(eci, "review_time = 1342051200") ;
+ query(eci, "review_time > 1342051200") ;
+ query(eci, "review_score > 0");
+ query(eci, "review_score > 2");
+ query(eci, "review_score > 3");
+ query(eci, "review_score > 4");
+ query(eci, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityCollectionIndex eci, String query ) {;
+ Query q = Query.fromQL(query) ;
+ Results results = eci.execute( q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d2cb960f/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
new file mode 100644
index 0000000..3c2390b
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.index.guice;
+
+import org.apache.usergrid.persistence.collection.guice.TestModule;
+
+
+public class TestIndexModule extends TestModule {
+
+ @Override
+ protected void configure() {
+ install( new IndexModule() );
+ }
+}
[10/18] git commit: Merge branch 'two-dot-o' of
github.com:usergrid/usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of github.com:usergrid/usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/56a2e214
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/56a2e214
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/56a2e214
Branch: refs/heads/hystrix-integration
Commit: 56a2e214d4a529c5f7b908c52d1dc7f4e1c13e99
Parents: 1b5a581 9d6ed41
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 21 15:54:20 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 21 15:54:20 2014 -0700
----------------------------------------------------------------------
.gitignore | 6 +
stack/awscluster/README.md | 31 +-
stack/awscluster/cassandra-cf.json | 315 ---------
stack/awscluster/pom.xml | 6 +-
.../main/dist/init_instance/init_db_server.sh | 54 ++
.../main/dist/init_instance/init_instance.sh | 54 --
.../main/dist/init_instance/init_rest_server.sh | 41 ++
.../dist/init_instance/install_cassandra.sh | 63 +-
stack/awscluster/src/main/dist/update.sh | 16 +
.../src/main/groovy/configure_cassandra.groovy | 2 +-
.../src/main/groovy/configure_portal_new.groovy | 17 +
.../src/main/groovy/configure_usergrid.groovy | 126 ++++
stack/awscluster/ugcluster-cf.json | 647 +++++++++++++++++++
.../serialization/SerializationFig.java | 5 +
...MvccEntitySerializationStrategyImplTest.java | 57 +-
stack/corepersistence/perftest1/pom.xml | 76 +++
.../usergrid/persistence/UsergridBootstrap.java | 303 +++++++++
.../src/main/resources/log4j.properties | 44 ++
.../src/main/resources/project.properties | 1 +
.../main/resources/usergrid-test-context.xml | 39 ++
.../persistence/Usergrid1PerformanceTest.java | 302 +++++++++
.../persistence/UsergridBootstrapTest.java | 56 ++
stack/corepersistence/perftest2/pom.xml | 64 ++
.../main/groovy/perftest2/CreateEntity.groovy | 100 +++
.../persistence/CorePerformanceTest.java | 318 +++++++++
.../index/guice/TestIndexModule.java | 30 +
.../usergrid/persistence/index/IndexFig.java | 8 +-
.../index/impl/EsEntityCollectionIndex.java | 16 +-
.../persistence/index/impl/EsProvider.java | 53 +-
.../index/legacy/EntityManagerFacade.java | 200 ++++++
.../apache/usergrid/utils/EntityBuilder.java | 177 +++++
.../index/impl/CorePerformanceIT.java | 304 +++++++++
.../index/impl/EntityCollectionIndexTest.java | 2 +-
.../persistence/index/legacy/EntityBuilder.java | 177 -----
.../index/legacy/EntityManagerFacade.java | 200 ------
.../src/test/resources/dynamic-test.properties | 2 +-
.../src/test/resources/usergrid-CHOP.properties | 5 +-
.../src/test/resources/usergrid-UNIT.properties | 2 +-
stack/pom.xml | 4 +-
39 files changed, 3054 insertions(+), 869 deletions(-)
----------------------------------------------------------------------
[11/18] git commit: Fixes bug in query for wait script
Posted by sn...@apache.org.
Fixes bug in query for wait script
Fixed typo
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a705630f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a705630f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a705630f
Branch: refs/heads/hystrix-integration
Commit: a705630f6e927b70452c22f426c746bc94b9ab71
Parents: 56a2e21
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 21 15:54:49 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 21 15:55:21 2014 -0700
----------------------------------------------------------------------
stack/awscluster/src/main/groovy/wait_for_instances.groovy | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a705630f/stack/awscluster/src/main/groovy/wait_for_instances.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/wait_for_instances.groovy b/stack/awscluster/src/main/groovy/wait_for_instances.groovy
index 4f125b6..a157ceb 100644
--- a/stack/awscluster/src/main/groovy/wait_for_instances.groovy
+++ b/stack/awscluster/src/main/groovy/wait_for_instances.groovy
@@ -24,7 +24,7 @@ println "Waiting..."
def count = 0
while (true) {
try {
- def selectResult = sdbClient.select(new SelectRequest((String)"select * from ${domain}"))
+ def selectResult = sdbClient.select(new SelectRequest((String)"select * from `${domain}`"))
for (item in selectResult.getItems()) {
def att = item.getAttributes().get(0)
if (att.getValue().equals(stackName)) {
[16/18] git commit: Merge branch 'two-dot-o' of
github.com:usergrid/usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of github.com:usergrid/usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fc105970
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fc105970
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fc105970
Branch: refs/heads/two-dot-o
Commit: fc105970fa93f486b24cc1fbfc0df85349d96f97
Parents: a4dd97a c4f9d0f
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 24 08:16:24 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 24 08:16:24 2014 -0700
----------------------------------------------------------------------
stack/corepersistence/perftest1/pom.xml | 9 ++---
.../persistence/Usergrid1PerformanceTest.java | 36 +++++++++++---------
2 files changed, 22 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
[08/18] git commit: Updating to reflect current state of awscluster.
Posted by sn...@apache.org.
Updating to reflect current state of awscluster.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9d6ed417
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9d6ed417
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9d6ed417
Branch: refs/heads/hystrix-integration
Commit: 9d6ed417e3773e711ae2adb0bb38fcf99b72413f
Parents: c3e0bf8
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Mar 21 11:53:53 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Mar 21 11:53:53 2014 -0400
----------------------------------------------------------------------
stack/awscluster/README.md | 31 ++++++++++++-------------------
1 file changed, 12 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9d6ed417/stack/awscluster/README.md
----------------------------------------------------------------------
diff --git a/stack/awscluster/README.md b/stack/awscluster/README.md
index 1f3f7f5..b7558b6 100644
--- a/stack/awscluster/README.md
+++ b/stack/awscluster/README.md
@@ -4,12 +4,12 @@ AWS Cluster
*UNDER CONSTRUCTION*
This project provides a AWS Cloud Formation template that launches a complete Usergrid installation
-on Amazon EC2. Currently it launches a set of instances running Cassanrda, Priam and Elastic Search.
+on Amazon EC2. Currently it launches a set of instances running Cassanrda Elastic Search.
Two parts:
1) A Maven assembly that builds an installer tarball and uploads it to S3.
-The tarball includes scripts to install Oracle JDK, Cassandra and Priam on EC2 instance.
+The tarball includes scripts to install Oracle JDK, Cassandra an EC2 instance.
2) A CloudFormation script that creates an autoscaling cluster, security group and some
number of EC2 instances, The template includes a CloudInit script that runs scipts from
@@ -18,26 +18,21 @@ the installer tarball to configure and start Cassandra on each node.
Getting set-up
---
-To setup our AWS account to use PriamCluster here's what you need to do:
+To setup our AWS account to use AWS Cluster here's what you need to do:
* __Create an AWS EC2 key pair__. You will need this PEM file to login to your Cassandra instances.
-* __Create an AWS S3 bucket__ for the PriamCluster install bundle. Create an S3 bucket in your
-account with the name `ug-cloudformation-priam`.
+* __Create an AWS S3 bucket__ for the AWS Cluster install bundle. e.g. Create an S3 bucket in your
+account with the name `ug-cloudformation`.
* __Upload the Oracle Java JDK__ to that same S3 bucket. Best practice is to use the Oracle Java u
JDK with Cassandra, so you must upload the JDK to the S3 bucket that we created above. The JDK must
be named `jdk-7u45-linux-x64.gz`.
-* __Create AWS SimpleDB domain for Priam__ properties and instance registration. It's best if you
-use a SimpleDB client (e.g. SDBNavigator for Chrome) to create these domains. You must create two
-domains named `PriamProperties` and `InstanceIdentity`. Just create the domains and Priam will create
-the attributes.
-
* __Create an aws.properties file__ with your AWS credentials in the same directory as this
README file. The file is git-ignored so you don't have to worry about accidentally committing it.
-* __Deploy this the PriamCluster assembly__ by running the Maven command `mvn depoy` in the same
+* __Deploy this the AWS Cluster assembly__ by running the Maven command `mvn depoy` in the same
directory as this README file.
@@ -47,8 +42,8 @@ Login to AWS Console and create a new CloudFormation stack. On the first screen,
and simple name for your stack and choose the option to upload a template and upload
the `cassandra-cf.json` file in this very directory.
-On the next screen, enter the number of servers you wish to create, the replication factor and
-the instance type you wish to use. Check the "I acknowlege that this will create IAM resources"
+On the next screen, enter the number of DB and REST servers you wish to create, the replication factor and
+the instance type you wish to use. Check the "I acknowledge that this will create IAM resources"
and then click Next, Next, Next to take defaults and start the stack.
Watch the EC2 console and see your instances come up. They will be tagged with the stack name
@@ -71,14 +66,12 @@ Here's what happens when the stack is started.
CloudFormation reads the `cassanrda-cf.json` template and uses starts the EC2 instances that it
specifies. When each instance comes up CloudFormation runs the CloudInit script specified in
-`cassandra-cf.json` and that script downloads the PriamCluster, sets up some environment scripts
-and calls the `init_instance/init_cass.sh` script to complete the setup.
+`ugcluster-cf.json` and that script downloads the AWS Cluster, sets up some environment scripts
+and calls the `init_instance/init_db_server.sh` script to complete the setup.
-The `init_instance/init_cass.sh` calls `install_oraclejdk.sh` to download the JDK from S3 and
+The `init_instance/init_db_server.sh` calls `install_oraclejdk.sh` to download the JDK from S3 and
install it. The script then installs Tomcat and Cassandra. Next it uses Groovy scripts to configure
-Priam and Cassandra and wait for other Cassandra nodes to come alive.
+Cassandra and wait for other Cassandra nodes to come alive.
-When Tomcat starts up, Priam comes to life. Priam reads it's SimpleDB tables, writes a new
-cassandra.yaml file and with the right seends and initial token starts Cassandra.
[03/18] git commit: Fixing annotation compile error.
Posted by sn...@apache.org.
Fixing annotation compile error.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5d5ee394
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5d5ee394
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5d5ee394
Branch: refs/heads/hystrix-integration
Commit: 5d5ee39466ec83bcfce1170537672f4e5ba6e96a
Parents: 6b7ffbe
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Mar 20 13:49:36 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Mar 20 13:49:36 2014 -0400
----------------------------------------------------------------------
.../apache/usergrid/persistence/index/impl/CorePerformanceIT.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5d5ee394/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
index 0008304..827e917 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
@@ -73,7 +73,7 @@ public class CorePerformanceIT {
@Ignore
@Test
- public void loadAndReadData( String[] args ) throws IOException, InterruptedException {
+ public void loadAndReadData() throws IOException, InterruptedException {
ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
Injector injector = Guice.createInjector( new TestIndexModule() );
[04/18] git commit: Exclude core persistence from RAT (for now).
Posted by sn...@apache.org.
Exclude core persistence from RAT (for now).
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e2f6fbfb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e2f6fbfb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e2f6fbfb
Branch: refs/heads/hystrix-integration
Commit: e2f6fbfbc9cc26be21efce87b24062f052d97fe3
Parents: 5d5ee39
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Mar 20 14:31:27 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Mar 20 14:31:27 2014 -0400
----------------------------------------------------------------------
stack/pom.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e2f6fbfb/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 93d57a0..1bc60c7 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -1799,6 +1799,9 @@
<exclude>**/cloudbees.xml</exclude>
<exclude>**/aws.properties</exclude>
+ <!-- TODO: need to add headers to many files in corepersistence -->
+ <exclude>**/corepersistence/**</exclude>
+
</excludes>
</configuration>
[07/18] git commit: Typo.
Posted by sn...@apache.org.
Typo.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c3e0bf8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c3e0bf8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c3e0bf8d
Branch: refs/heads/hystrix-integration
Commit: c3e0bf8d87361543716059bb16562101c95e0554
Parents: 4cf38be
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Mar 21 10:39:47 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Mar 21 10:39:47 2014 -0400
----------------------------------------------------------------------
.../persistence/Usegrid1PerformanceTest.java | 302 -------------------
.../persistence/Usergrid1PerformanceTest.java | 302 +++++++++++++++++++
2 files changed, 302 insertions(+), 302 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3e0bf8d/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
deleted file mode 100644
index a818199..0000000
--- a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.commons.lang.NumberUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * TODO: make configurable, add CHOP markup.
- */
-public class Usegrid1PerformanceTest {
- private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceTest.class);
-
- // max entities we will write and read
- static int maxEntities = Integer.MAX_VALUE;
-
- // each app will get all data
- static int orgCount = 2;
- static int appCount = 5 ;
-
- // number of threads = orgCount x appCount
-
- // total number of records = orgCount x appCount x numRecords
-
- private final EntityManagerFactory emf;
-
-
- public Usegrid1PerformanceTest() throws Throwable {
- emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
- }
-
-
- public EntityManagerFactory getEmf() {
- return emf;
- }
-
-
- @Test
- public void loadAndReadData() throws Exception {
-
- log.info("Start Data Load");
- List<UUID> apps = loadData();
- log.info("Finish Data Load");
-
- log.info("Start Data Read");
- readData( apps );
- log.info("Finish Data Read");
-
- runSelectedQueries( apps );
-
- }
-
-
- private List<UUID> loadData() throws Exception {
-
- long time = new Date().getTime();
-
- List<UUID> apps = new ArrayList<UUID>();
- List<Thread> threads = new ArrayList<Thread>();
-
- for ( int i=0; i<orgCount; i++ ) {
-
- for ( int j=0; j<appCount; j++ ) {
-
- UUID appId = getEmf().createApplication(
- "testorg-" + RandomStringUtils.randomAlphanumeric(6),
- "testapp-" + RandomStringUtils.randomAlphanumeric(6));
-
- apps.add( appId );
-
- Thread t = new Thread( new DataLoader( appId ));
- t.start();
- threads.add(t);
- }
- }
-
- // wait for indexing to end
- for ( Thread t : threads ) {
- t.join();
- }
-
- return apps;
- }
-
-
- private void readData( List<UUID> apps ) throws InterruptedException {
-
- List<Thread> threads = new ArrayList<Thread>();
- for ( UUID app : apps ) {
-
- Thread t = new Thread( new DataReader( app ));
- t.start();
- threads.add(t);
- }
-
- // wait for reading to end
- for ( Thread t : threads ) {
- t.join();
- }
- }
-
-
- private class DataReader implements Runnable {
- UUID app;
-
- public DataReader( UUID app ) {
- this.app = app;
- }
-
- public void run() {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- UUID appId = app;
-
- Query query = Query.fromQL( "review_score > 0"); // get all reviews;
- query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
-
- Results results;
- try {
- results = em.searchCollection( em.getApplicationRef(), "reviews", query );
- } catch (Exception ex) {
- log.error("Error on search, aborting", ex);
- return;
- }
-
- results.getEntities(); // cause retrieval from Cassandra
- int count = results.size();
-
- while ( results.hasCursor() && count < maxEntities ) {
- query.setCursor( results.getCursor() ) ;
- try {
- results = em.searchCollection( em.getApplicationRef(), "reviews", query );
- } catch (Exception ex) {
- log.error("Error on search, aborting", ex);
- log.info("Read {} reviews in {}", count, appId );
- return;
- }
- results.getEntities(); // cause retrieval from Cassanda;
- count += results.size();
-
- log.info("Read {} reviews in {}", count, appId );
- }
- }
- }
-
-
- private class DataLoader implements Runnable {
- UUID app;
-
- public DataLoader( UUID scope ) {
- this.app = scope;
- }
-
- public void run() {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- BufferedReader br;
- try {
- InputStreamReader isr = new InputStreamReader(
- getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
- br = new BufferedReader(isr);
- } catch (Exception ex) {
- throw new RuntimeException("Error opening file", ex);
- }
- String s = null;
-
- // create the first entry
- Map<String, Object> currentEntityMap = new HashMap<String, Object>();
-
- UUID appId = app;
-
- int count = 0;
- try {
- while ( (s = br.readLine()) != null && count < maxEntities ) {
-
- try {
-
- if ( s.trim().equals("")) { // then we are at end of a record
-
- // write and index current entity
- Entity entity = em.create("review", currentEntityMap );
-
- if ( maxEntities < 20 ) {
- log.info("Index written for {}", entity.getUuid());
- log.info("---");
- }
-
- // create the next entity
- currentEntityMap = new HashMap<String, Object>();
-
- count++;
- if (count % 100000 == 0) {
- log.info("Indexed {} reviews in {}", count, appId );
- }
- continue;
- }
-
- // process a field
- String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
- String value = s.substring( s.indexOf(":") + 1 ).trim();
-
- if ( maxEntities < 20 ) {
- log.info("Indexing {} = {}", name, value);
- }
-
- if ( NumberUtils.isNumber(value) && value.contains(".")) {
- currentEntityMap.put( name, Double.parseDouble(value));
-
- } else if ( NumberUtils.isNumber(value) ) {
- currentEntityMap.put( name, Long.parseLong(value));
-
- } else {
- currentEntityMap.put( name, value.toString() );
- }
-
- } catch ( Exception e ) {
- log.info("Error on line " + count);
- }
- }
-
- } catch (IOException ex) {
- throw new RuntimeException("Error reading file", ex);
- }
- }
- }
-
-
- public void runSelectedQueries( List<UUID> apps ) throws Exception {
-
- for ( UUID app : apps ) {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- query(em, "product_productid = 'B006K2ZZ7K'") ;
- query(em, "review_profilename = 'Twoapennything'") ;
- query(em, "review_profilename contains 'Natalia'") ;
- query(em, "review_profilename contains 'Patrick'") ;
- query(em, "review_time = 1342051200") ;
- query(em, "review_time > 1342051200") ;
- query(em, "review_score > 0");
- query(em, "review_score > 2");
- query(em, "review_score > 3");
- query(em, "review_score > 4");
- query(em, "review_score > 5");
- }
- }
-
- public static void query( EntityManager em, String query ) throws Exception {
- Query q = Query.fromQL(query) ;
- Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
- log.info("size = {} returned from query {}",results.size(), q.getQl() );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3e0bf8d/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
new file mode 100644
index 0000000..a818199
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.NumberUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class Usegrid1PerformanceTest {
+ private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceTest.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityManagerFactory emf;
+
+
+ public Usegrid1PerformanceTest() throws Throwable {
+ emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
+ }
+
+
+ public EntityManagerFactory getEmf() {
+ return emf;
+ }
+
+
+ @Test
+ public void loadAndReadData() throws Exception {
+
+ log.info("Start Data Load");
+ List<UUID> apps = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( apps );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( apps );
+
+ }
+
+
+ private List<UUID> loadData() throws Exception {
+
+ long time = new Date().getTime();
+
+ List<UUID> apps = new ArrayList<UUID>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ UUID appId = getEmf().createApplication(
+ "testorg-" + RandomStringUtils.randomAlphanumeric(6),
+ "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+ apps.add( appId );
+
+ Thread t = new Thread( new DataLoader( appId ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return apps;
+ }
+
+
+ private void readData( List<UUID> apps ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( UUID app : apps ) {
+
+ Thread t = new Thread( new DataReader( app ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+
+ private class DataReader implements Runnable {
+ UUID app;
+
+ public DataReader( UUID app ) {
+ this.app = app;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ UUID appId = app;
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ return;
+ }
+
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ log.info("Read {} reviews in {}", count, appId );
+ return;
+ }
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {}", count, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ UUID app;
+
+ public DataLoader( UUID scope ) {
+ this.app = scope;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Map<String, Object> currentEntityMap = new HashMap<String, Object>();
+
+ UUID appId = app;
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ Entity entity = em.create("review", currentEntityMap );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", entity.getUuid());
+ log.info("---");
+ }
+
+ // create the next entity
+ currentEntityMap = new HashMap<String, Object>();
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {}", count, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ currentEntityMap.put( name, Double.parseDouble(value));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ currentEntityMap.put( name, Long.parseLong(value));
+
+ } else {
+ currentEntityMap.put( name, value.toString() );
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+ }
+ }
+
+
+ public void runSelectedQueries( List<UUID> apps ) throws Exception {
+
+ for ( UUID app : apps ) {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ query(em, "product_productid = 'B006K2ZZ7K'") ;
+ query(em, "review_profilename = 'Twoapennything'") ;
+ query(em, "review_profilename contains 'Natalia'") ;
+ query(em, "review_profilename contains 'Patrick'") ;
+ query(em, "review_time = 1342051200") ;
+ query(em, "review_time > 1342051200") ;
+ query(em, "review_score > 0");
+ query(em, "review_score > 2");
+ query(em, "review_score > 3");
+ query(em, "review_score > 4");
+ query(em, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityManager em, String query ) throws Exception {
+ Query q = Query.fromQL(query) ;
+ Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}