You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/21 20:21:02 UTC
[15/44] git commit: Minor fixes, clean-ups.
Minor fixes, clean-ups.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4cf38be6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4cf38be6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4cf38be6
Branch: refs/pull/77/head
Commit: 4cf38be6d8401bef360e6b13cd2ab0360f51683d
Parents: d2cb960
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Mar 21 10:16:12 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Mar 21 10:16:12 2014 -0400
----------------------------------------------------------------------
stack/corepersistence/perftest1/pom.xml | 21 --
.../persistence/Usegrid1PerformanceIT.java | 304 ------------------
.../persistence/Usegrid1PerformanceTest.java | 302 +++++++++++++++++
stack/corepersistence/perftest2/pom.xml | 21 --
.../usergrid/persistence/CorePerformanceIT.java | 321 -------------------
.../persistence/CorePerformanceTest.java | 318 ++++++++++++++++++
6 files changed, 620 insertions(+), 667 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest1/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/pom.xml b/stack/corepersistence/perftest1/pom.xml
index abee483..0e380b1 100644
--- a/stack/corepersistence/perftest1/pom.xml
+++ b/stack/corepersistence/perftest1/pom.xml
@@ -71,27 +71,6 @@
<extensions>true</extensions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
- <overWriteReleases>false</overWriteReleases>
- <overWriteSnapshots>false</overWriteSnapshots>
- <overWriteIfNewer>true</overWriteIfNewer>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
</plugins>
</build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
deleted file mode 100644
index af47a8c..0000000
--- a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceIT.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.commons.lang.NumberUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * TODO: make configurable, add CHOP markup.
- */
-public class Usegrid1PerformanceIT {
- private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceIT.class);
-
- // max entities we will write and read
- static int maxEntities = Integer.MAX_VALUE;
-
- // each app will get all data
- static int orgCount = 2;
- static int appCount = 5 ;
-
- // number of threads = orgCount x appCount
-
- // total number of records = orgCount x appCount x numRecords
-
- private final EntityManagerFactory emf;
-
-
- public Usegrid1PerformanceIT() throws Throwable {
- emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
- }
-
-
- public EntityManagerFactory getEmf() {
- return emf;
- }
-
-
- @Ignore
- @Test
- public void loadAndReadData() throws Exception {
-
- log.info("Start Data Load");
- List<UUID> apps = loadData();
- log.info("Finish Data Load");
-
- log.info("Start Data Read");
- readData( apps );
- log.info("Finish Data Read");
-
- runSelectedQueries( apps );
-
- }
-
-
- private List<UUID> loadData() throws Exception {
-
- long time = new Date().getTime();
-
- List<UUID> apps = new ArrayList<UUID>();
- List<Thread> threads = new ArrayList<Thread>();
-
- for ( int i=0; i<orgCount; i++ ) {
-
- for ( int j=0; j<appCount; j++ ) {
-
- UUID appId = getEmf().createApplication(
- "testorg-" + RandomStringUtils.randomAlphanumeric(6),
- "testapp-" + RandomStringUtils.randomAlphanumeric(6));
-
- apps.add( appId );
-
- Thread t = new Thread( new DataLoader( appId ));
- t.start();
- threads.add(t);
- }
- }
-
- // wait for indexing to end
- for ( Thread t : threads ) {
- t.join();
- }
-
- return apps;
- }
-
-
- private void readData( List<UUID> apps ) throws InterruptedException {
-
- List<Thread> threads = new ArrayList<Thread>();
- for ( UUID app : apps ) {
-
- Thread t = new Thread( new DataReader( app ));
- t.start();
- threads.add(t);
- }
-
- // wait for reading to end
- for ( Thread t : threads ) {
- t.join();
- }
- }
-
-
- private class DataReader implements Runnable {
- UUID app;
-
- public DataReader( UUID app ) {
- this.app = app;
- }
-
- public void run() {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- UUID appId = app;
-
- Query query = Query.fromQL( "review_score > 0"); // get all reviews;
- query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
-
- Results results;
- try {
- results = em.searchCollection( em.getApplicationRef(), "reviews", query );
- } catch (Exception ex) {
- log.error("Error on search, aborting", ex);
- return;
- }
-
- results.getEntities(); // cause retrieval from Cassandra
- int count = results.size();
-
- while ( results.hasCursor() && count < maxEntities ) {
- query.setCursor( results.getCursor() ) ;
- try {
- results = em.searchCollection( em.getApplicationRef(), "reviews", query );
- } catch (Exception ex) {
- log.error("Error on search, aborting", ex);
- log.info("Read {} reviews in {}", count, appId );
- return;
- }
- results.getEntities(); // cause retrieval from Cassanda;
- count += results.size();
-
- log.info("Read {} reviews in {}", count, appId );
- }
- }
- }
-
-
- private class DataLoader implements Runnable {
- UUID app;
-
- public DataLoader( UUID scope ) {
- this.app = scope;
- }
-
- public void run() {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- BufferedReader br;
- try {
- InputStreamReader isr = new InputStreamReader(
- getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
- br = new BufferedReader(isr);
- } catch (Exception ex) {
- throw new RuntimeException("Error opening file", ex);
- }
- String s = null;
-
- // create the first entry
- Map<String, Object> currentEntityMap = new HashMap<String, Object>();
-
- UUID appId = app;
-
- int count = 0;
- try {
- while ( (s = br.readLine()) != null && count < maxEntities ) {
-
- try {
-
- if ( s.trim().equals("")) { // then we are at end of a record
-
- // write and index current entity
- Entity entity = em.create("review", currentEntityMap );
-
- if ( maxEntities < 20 ) {
- log.info("Index written for {}", entity.getUuid());
- log.info("---");
- }
-
- // create the next entity
- currentEntityMap = new HashMap<String, Object>();
-
- count++;
- if (count % 100000 == 0) {
- log.info("Indexed {} reviews in {}", count, appId );
- }
- continue;
- }
-
- // process a field
- String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
- String value = s.substring( s.indexOf(":") + 1 ).trim();
-
- if ( maxEntities < 20 ) {
- log.info("Indexing {} = {}", name, value);
- }
-
- if ( NumberUtils.isNumber(value) && value.contains(".")) {
- currentEntityMap.put( name, Double.parseDouble(value));
-
- } else if ( NumberUtils.isNumber(value) ) {
- currentEntityMap.put( name, Long.parseLong(value));
-
- } else {
- currentEntityMap.put( name, value.toString() );
- }
-
- } catch ( Exception e ) {
- log.info("Error on line " + count);
- }
- }
-
- } catch (IOException ex) {
- throw new RuntimeException("Error reading file", ex);
- }
- }
- }
-
-
- public void runSelectedQueries( List<UUID> apps ) throws Exception {
-
- for ( UUID app : apps ) {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- query(em, "product_productid = 'B006K2ZZ7K'") ;
- query(em, "review_profilename = 'Twoapennything'") ;
- query(em, "review_profilename contains 'Natalia'") ;
- query(em, "review_profilename contains 'Patrick'") ;
- query(em, "review_time = 1342051200") ;
- query(em, "review_time > 1342051200") ;
- query(em, "review_score > 0");
- query(em, "review_score > 2");
- query(em, "review_score > 3");
- query(em, "review_score > 4");
- query(em, "review_score > 5");
- }
- }
-
- public static void query( EntityManager em, String query ) throws Exception {
- Query q = Query.fromQL(query) ;
- Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
- log.info("size = {} returned from query {}",results.size(), q.getQl() );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
new file mode 100644
index 0000000..a818199
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.NumberUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class Usegrid1PerformanceTest {
+ private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceTest.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityManagerFactory emf;
+
+
+ public Usegrid1PerformanceTest() throws Throwable {
+ emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
+ }
+
+
+ public EntityManagerFactory getEmf() {
+ return emf;
+ }
+
+
+ @Test
+ public void loadAndReadData() throws Exception {
+
+ log.info("Start Data Load");
+ List<UUID> apps = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( apps );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( apps );
+
+ }
+
+
+ private List<UUID> loadData() throws Exception {
+
+ long time = new Date().getTime();
+
+ List<UUID> apps = new ArrayList<UUID>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ UUID appId = getEmf().createApplication(
+ "testorg-" + RandomStringUtils.randomAlphanumeric(6),
+ "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+ apps.add( appId );
+
+ Thread t = new Thread( new DataLoader( appId ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return apps;
+ }
+
+
+ private void readData( List<UUID> apps ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( UUID app : apps ) {
+
+ Thread t = new Thread( new DataReader( app ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+
+ private class DataReader implements Runnable {
+ UUID app;
+
+ public DataReader( UUID app ) {
+ this.app = app;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ UUID appId = app;
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ return;
+ }
+
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ log.info("Read {} reviews in {}", count, appId );
+ return;
+ }
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {}", count, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ UUID app;
+
+ public DataLoader( UUID scope ) {
+ this.app = scope;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Map<String, Object> currentEntityMap = new HashMap<String, Object>();
+
+ UUID appId = app;
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ Entity entity = em.create("review", currentEntityMap );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", entity.getUuid());
+ log.info("---");
+ }
+
+ // create the next entity
+ currentEntityMap = new HashMap<String, Object>();
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {}", count, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ currentEntityMap.put( name, Double.parseDouble(value));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ currentEntityMap.put( name, Long.parseLong(value));
+
+ } else {
+ currentEntityMap.put( name, value.toString() );
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+ }
+ }
+
+
+ public void runSelectedQueries( List<UUID> apps ) throws Exception {
+
+ for ( UUID app : apps ) {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ query(em, "product_productid = 'B006K2ZZ7K'") ;
+ query(em, "review_profilename = 'Twoapennything'") ;
+ query(em, "review_profilename contains 'Natalia'") ;
+ query(em, "review_profilename contains 'Patrick'") ;
+ query(em, "review_time = 1342051200") ;
+ query(em, "review_time > 1342051200") ;
+ query(em, "review_score > 0");
+ query(em, "review_score > 2");
+ query(em, "review_score > 3");
+ query(em, "review_score > 4");
+ query(em, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityManager em, String query ) throws Exception {
+ Query q = Query.fromQL(query) ;
+ Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest2/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/pom.xml b/stack/corepersistence/perftest2/pom.xml
index be1a457..34ef45f 100644
--- a/stack/corepersistence/perftest2/pom.xml
+++ b/stack/corepersistence/perftest2/pom.xml
@@ -59,27 +59,6 @@
<extensions>true</extensions>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
- <executions>
- <execution>
- <id>copy-dependencies</id>
- <phase>package</phase>
- <goals>
- <goal>copy-dependencies</goal>
- </goals>
- <configuration>
- <outputDirectory>${project.build.directory}/alternateLocation</outputDirectory>
- <overWriteReleases>false</overWriteReleases>
- <overWriteSnapshots>false</overWriteSnapshots>
- <overWriteIfNewer>true</overWriteIfNewer>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
</plugins>
</build>
</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
deleted file mode 100644
index 8536fca..0000000
--- a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceIT.java
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence;
-
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.netflix.config.ConfigurationManager;
-import java.io.BufferedReader;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import org.apache.commons.lang3.math.NumberUtils;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.index.EntityCollectionIndex;
-import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory;
-import org.apache.usergrid.persistence.index.guice.TestIndexModule;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.DoubleField;
-import org.apache.usergrid.persistence.model.field.LongField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.query.Query;
-import org.apache.usergrid.persistence.query.Results;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * TODO: make configurable, add CHOP markup.
- */
-public class CorePerformanceIT {
- private static final Logger log = LoggerFactory.getLogger(CorePerformanceIT.class);
-
- // max entities we will write and read
- static int maxEntities = Integer.MAX_VALUE;
-
- // each app will get all data
- static int orgCount = 2;
- static int appCount = 5 ;
-
- // number of threads = orgCount x appCount
-
- // total number of records = orgCount x appCount x numRecords
-
- private final EntityCollectionManagerFactory ecmf;
- private final EntityCollectionIndexFactory ecif;
-
- public CorePerformanceIT() {
- Injector injector = Guice.createInjector( new TestIndexModule() );
- ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
- ecif = injector.getInstance( EntityCollectionIndexFactory.class );
- }
-
- @Ignore
- @Test
- public void loadAndReadData() throws IOException, InterruptedException {
-
- ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
-
- // only on first run
- //MigrationManager m = injector.getInstance( MigrationManager.class )
- //m.migrate()
-
- log.info("Start Data Load");
- List<CollectionScope> scopes = loadData();
- log.info("Finish Data Load");
-
- log.info("Start Data Read");
- readData( scopes );
- log.info("Finish Data Read");
-
- runSelectedQueries( scopes );
-
- }
-
-
- private List<CollectionScope> loadData() throws InterruptedException {
-
- long time = new Date().getTime();
-
- List<CollectionScope> scopes = new ArrayList<CollectionScope>();
- List<Thread> threads = new ArrayList<Thread>();
-
- for ( int i=0; i<orgCount; i++ ) {
-
- String orgName = "org-" + i + "-" + time;
- final Id orgId = new SimpleId(orgName);
-
- for ( int j=0; j<appCount; j++ ) {
-
- String appName = "app-" + j + "-" + time;
- final Id appId = new SimpleId(appName);
-
- CollectionScope scope = new CollectionScopeImpl( orgId, appId, "reviews" );
- scopes.add( scope );
-
- Thread t = new Thread( new DataLoader( scope ));
- t.start();
- threads.add(t);
- }
- }
-
- // wait for indexing to end
- for ( Thread t : threads ) {
- t.join();
- }
-
- return scopes;
- }
-
-
- private void readData( List<CollectionScope> scopes ) throws InterruptedException {
-
- List<Thread> threads = new ArrayList<Thread>();
- for ( CollectionScope scope : scopes ) {
-
- Thread t = new Thread( new DataReader( scope ));
- t.start();
- threads.add(t);
- }
-
- // wait for reading to end
- for ( Thread t : threads ) {
- t.join();
- }
- }
-
- /**
- * @return the ecmf
- */
- public EntityCollectionManagerFactory getEcmf() {
- return ecmf;
- }
-
- /**
- * @return the ecif
- */
- public EntityCollectionIndexFactory getEcif() {
- return ecif;
- }
-
-
- private class DataReader implements Runnable {
- CollectionScope scope;
-
- public DataReader( CollectionScope scope ) {
- this.scope = scope;
- }
-
- public void run() {
-
- Id orgId = scope.getOrganization();
- Id appId = scope.getOwner();
-
- EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
- EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
-
- Query query = Query.fromQL( "review_score > 0"); // get all reviews;
- query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
-
- Results results = eci.execute( query );
- results.getEntities(); // cause retrieval from Cassandra
- int count = results.size();
-
- while ( results.hasCursor() && count < maxEntities ) {
- query.setCursor( results.getCursor() ) ;
- results = eci.execute( query );
- results.getEntities(); // cause retrieval from Cassanda;
- count += results.size();
-
- log.info("Read {} reviews in {} / {} ", count, orgId, appId );
- }
- }
- }
-
-
- private class DataLoader implements Runnable {
- CollectionScope scope;
-
- public DataLoader( CollectionScope scope ) {
- this.scope = scope;
- }
-
- public void run() {
-
- EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
- EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
-
- BufferedReader br;
- try {
- InputStreamReader isr = new InputStreamReader(
- getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
- br = new BufferedReader(isr);
- } catch (Exception ex) {
- throw new RuntimeException("Error opening file", ex);
- }
- String s = null;
-
- // create the first entry
- Entity current = new Entity(
- new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
-
- Id orgId = scope.getOrganization();
- Id appId = scope.getOwner();
-
- int count = 0;
- try {
- while ( (s = br.readLine()) != null && count < maxEntities ) {
-
- try {
-
- if ( s.trim().equals("")) { // then we are at end of a record
-
- // write and index current entity
- ecm.write( current ).toBlockingObservable().last();
- eci.index( current );
-
- if ( maxEntities < 20 ) {
- log.info("Index written for {}", current.getId());
- log.info("---");
- }
-
- // create the next entity
- current = new Entity(
- new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
-
- count++;
- if (count % 100000 == 0) {
- log.info("Indexed {} reviews in {} / {} ", count, orgId, appId );
- }
- continue;
- }
-
- // process a field
- String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
- String value = s.substring( s.indexOf(":") + 1 ).trim();
-
- if ( maxEntities < 20 ) {
- log.info("Indexing {} = {}", name, value);
- }
-
- if ( NumberUtils.isNumber(value) && value.contains(".")) {
- current.setField( new DoubleField( name, Double.parseDouble(value)));
-
- } else if ( NumberUtils.isNumber(value) ) {
- current.setField( new LongField( name, Long.parseLong(value)));
-
- } else {
- current.setField( new StringField( name, value.toString() ));
- }
-
- } catch ( Exception e ) {
- log.info("Error on line " + count);
- }
- }
-
- } catch (IOException ex) {
- throw new RuntimeException("Error reading file", ex);
- }
-
- eci.refresh();
- }
- }
-
-
- public void runSelectedQueries( List<CollectionScope> scopes ) {
-
- for ( CollectionScope scope : scopes ) {
-
- EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
- EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
-
- // TODO: come up with more and more complex queries for CorePerformanceIT
-
- query(eci, "product_productid = 'B006K2ZZ7K'") ;
- query(eci, "review_profilename = 'Twoapennything'") ;
- query(eci, "review_profilename contains 'Natalia'") ;
- query(eci, "review_profilename contains 'Patrick'") ;
- query(eci, "review_time = 1342051200") ;
- query(eci, "review_time > 1342051200") ;
- query(eci, "review_score > 0");
- query(eci, "review_score > 2");
- query(eci, "review_score > 3");
- query(eci, "review_score > 4");
- query(eci, "review_score > 5");
- }
- }
-
- public static void query( EntityCollectionIndex eci, String query ) {;
- Query q = Query.fromQL(query) ;
- Results results = eci.execute( q );
- log.info("size = {} returned from query {}",results.size(), q.getQl() );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4cf38be6/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceTest.java b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceTest.java
new file mode 100644
index 0000000..1e69167
--- /dev/null
+++ b/stack/corepersistence/perftest2/src/test/java/org/apache/usergrid/persistence/CorePerformanceTest.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.netflix.config.ConfigurationManager;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.index.EntityCollectionIndex;
+import org.apache.usergrid.persistence.index.EntityCollectionIndexFactory;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.persistence.query.Query;
+import org.apache.usergrid.persistence.query.Results;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class CorePerformanceTest {
+ private static final Logger log = LoggerFactory.getLogger(CorePerformanceTest.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityCollectionManagerFactory ecmf;
+ private final EntityCollectionIndexFactory ecif;
+
+ public CorePerformanceTest() {
+ Injector injector = Guice.createInjector( new TestIndexModule() );
+ ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
+ ecif = injector.getInstance( EntityCollectionIndexFactory.class );
+ }
+
+
+ @Test
+ public void loadAndReadData() throws IOException, InterruptedException {
+
+ ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
+
+ // only on first run
+ //MigrationManager m = injector.getInstance( MigrationManager.class )
+ //m.migrate()
+
+ log.info("Start Data Load");
+ List<CollectionScope> scopes = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( scopes );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( scopes );
+
+ }
+
+
+ private List<CollectionScope> loadData() throws InterruptedException {
+
+ long time = new Date().getTime();
+
+ List<CollectionScope> scopes = new ArrayList<CollectionScope>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ String orgName = "org-" + i + "-" + time;
+ final Id orgId = new SimpleId(orgName);
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ String appName = "app-" + j + "-" + time;
+ final Id appId = new SimpleId(appName);
+
+ CollectionScope scope = new CollectionScopeImpl( orgId, appId, "reviews" );
+ scopes.add( scope );
+
+ Thread t = new Thread( new DataLoader( scope ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return scopes;
+ }
+
+
+ private void readData( List<CollectionScope> scopes ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( CollectionScope scope : scopes ) {
+
+ Thread t = new Thread( new DataReader( scope ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+ /**
+ * @return the ecmf
+ */
+ public EntityCollectionManagerFactory getEcmf() {
+ return ecmf;
+ }
+
+ /**
+ * @return the ecif
+ */
+ public EntityCollectionIndexFactory getEcif() {
+ return ecif;
+ }
+
+
+ private class DataReader implements Runnable {
+ CollectionScope scope;
+
+ public DataReader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ results = eci.execute( query );
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {} / {} ", count, orgId, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ CollectionScope scope;
+
+ public DataLoader( CollectionScope scope ) {
+ this.scope = scope;
+ }
+
+ public void run() {
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Entity current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ Id orgId = scope.getOrganization();
+ Id appId = scope.getOwner();
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ ecm.write( current ).toBlockingObservable().last();
+ eci.index( current );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", current.getId());
+ log.info("---");
+ }
+
+ // create the next entity
+ current = new Entity(
+ new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {} / {} ", count, orgId, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ current.setField( new DoubleField( name, Double.parseDouble(value)));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ current.setField( new LongField( name, Long.parseLong(value)));
+
+ } else {
+ current.setField( new StringField( name, value.toString() ));
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+
+ eci.refresh();
+ }
+ }
+
+
+ public void runSelectedQueries( List<CollectionScope> scopes ) {
+
+ for ( CollectionScope scope : scopes ) {
+
+ EntityCollectionManager ecm = getEcmf().createCollectionManager( scope );
+ EntityCollectionIndex eci = getEcif().createCollectionIndex( scope );
+
+ // TODO: come up with more and more complex queries for CorePerformanceIT
+
+ query(eci, "product_productid = 'B006K2ZZ7K'") ;
+ query(eci, "review_profilename = 'Twoapennything'") ;
+ query(eci, "review_profilename contains 'Natalia'") ;
+ query(eci, "review_profilename contains 'Patrick'") ;
+ query(eci, "review_time = 1342051200") ;
+ query(eci, "review_time > 1342051200") ;
+ query(eci, "review_score > 0");
+ query(eci, "review_score > 2");
+ query(eci, "review_score > 3");
+ query(eci, "review_score > 4");
+ query(eci, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityCollectionIndex eci, String query ) {;
+ Query q = Query.fromQL(query) ;
+ Results results = eci.execute( q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}