You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/24 16:20:54 UTC
[07/18] git commit: Typo.
Typo.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c3e0bf8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c3e0bf8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c3e0bf8d
Branch: refs/heads/hystrix-integration
Commit: c3e0bf8d87361543716059bb16562101c95e0554
Parents: 4cf38be
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Mar 21 10:39:47 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Mar 21 10:39:47 2014 -0400
----------------------------------------------------------------------
.../persistence/Usegrid1PerformanceTest.java | 302 -------------------
.../persistence/Usergrid1PerformanceTest.java | 302 +++++++++++++++++++
2 files changed, 302 insertions(+), 302 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3e0bf8d/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
deleted file mode 100644
index a818199..0000000
--- a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usegrid1PerformanceTest.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.commons.lang.NumberUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * TODO: make configurable, add CHOP markup.
- */
-public class Usegrid1PerformanceTest {
- private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceTest.class);
-
- // max entities we will write and read
- static int maxEntities = Integer.MAX_VALUE;
-
- // each app will get all data
- static int orgCount = 2;
- static int appCount = 5 ;
-
- // number of threads = orgCount x appCount
-
- // total number of records = orgCount x appCount x numRecords
-
- private final EntityManagerFactory emf;
-
-
- public Usegrid1PerformanceTest() throws Throwable {
- emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
- }
-
-
- public EntityManagerFactory getEmf() {
- return emf;
- }
-
-
- @Test
- public void loadAndReadData() throws Exception {
-
- log.info("Start Data Load");
- List<UUID> apps = loadData();
- log.info("Finish Data Load");
-
- log.info("Start Data Read");
- readData( apps );
- log.info("Finish Data Read");
-
- runSelectedQueries( apps );
-
- }
-
-
- private List<UUID> loadData() throws Exception {
-
- long time = new Date().getTime();
-
- List<UUID> apps = new ArrayList<UUID>();
- List<Thread> threads = new ArrayList<Thread>();
-
- for ( int i=0; i<orgCount; i++ ) {
-
- for ( int j=0; j<appCount; j++ ) {
-
- UUID appId = getEmf().createApplication(
- "testorg-" + RandomStringUtils.randomAlphanumeric(6),
- "testapp-" + RandomStringUtils.randomAlphanumeric(6));
-
- apps.add( appId );
-
- Thread t = new Thread( new DataLoader( appId ));
- t.start();
- threads.add(t);
- }
- }
-
- // wait for indexing to end
- for ( Thread t : threads ) {
- t.join();
- }
-
- return apps;
- }
-
-
- private void readData( List<UUID> apps ) throws InterruptedException {
-
- List<Thread> threads = new ArrayList<Thread>();
- for ( UUID app : apps ) {
-
- Thread t = new Thread( new DataReader( app ));
- t.start();
- threads.add(t);
- }
-
- // wait for reading to end
- for ( Thread t : threads ) {
- t.join();
- }
- }
-
-
- private class DataReader implements Runnable {
- UUID app;
-
- public DataReader( UUID app ) {
- this.app = app;
- }
-
- public void run() {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- UUID appId = app;
-
- Query query = Query.fromQL( "review_score > 0"); // get all reviews;
- query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
-
- Results results;
- try {
- results = em.searchCollection( em.getApplicationRef(), "reviews", query );
- } catch (Exception ex) {
- log.error("Error on search, aborting", ex);
- return;
- }
-
- results.getEntities(); // cause retrieval from Cassandra
- int count = results.size();
-
- while ( results.hasCursor() && count < maxEntities ) {
- query.setCursor( results.getCursor() ) ;
- try {
- results = em.searchCollection( em.getApplicationRef(), "reviews", query );
- } catch (Exception ex) {
- log.error("Error on search, aborting", ex);
- log.info("Read {} reviews in {}", count, appId );
- return;
- }
- results.getEntities(); // cause retrieval from Cassanda;
- count += results.size();
-
- log.info("Read {} reviews in {}", count, appId );
- }
- }
- }
-
-
- private class DataLoader implements Runnable {
- UUID app;
-
- public DataLoader( UUID scope ) {
- this.app = scope;
- }
-
- public void run() {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- BufferedReader br;
- try {
- InputStreamReader isr = new InputStreamReader(
- getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
- br = new BufferedReader(isr);
- } catch (Exception ex) {
- throw new RuntimeException("Error opening file", ex);
- }
- String s = null;
-
- // create the first entry
- Map<String, Object> currentEntityMap = new HashMap<String, Object>();
-
- UUID appId = app;
-
- int count = 0;
- try {
- while ( (s = br.readLine()) != null && count < maxEntities ) {
-
- try {
-
- if ( s.trim().equals("")) { // then we are at end of a record
-
- // write and index current entity
- Entity entity = em.create("review", currentEntityMap );
-
- if ( maxEntities < 20 ) {
- log.info("Index written for {}", entity.getUuid());
- log.info("---");
- }
-
- // create the next entity
- currentEntityMap = new HashMap<String, Object>();
-
- count++;
- if (count % 100000 == 0) {
- log.info("Indexed {} reviews in {}", count, appId );
- }
- continue;
- }
-
- // process a field
- String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
- String value = s.substring( s.indexOf(":") + 1 ).trim();
-
- if ( maxEntities < 20 ) {
- log.info("Indexing {} = {}", name, value);
- }
-
- if ( NumberUtils.isNumber(value) && value.contains(".")) {
- currentEntityMap.put( name, Double.parseDouble(value));
-
- } else if ( NumberUtils.isNumber(value) ) {
- currentEntityMap.put( name, Long.parseLong(value));
-
- } else {
- currentEntityMap.put( name, value.toString() );
- }
-
- } catch ( Exception e ) {
- log.info("Error on line " + count);
- }
- }
-
- } catch (IOException ex) {
- throw new RuntimeException("Error reading file", ex);
- }
- }
- }
-
-
- public void runSelectedQueries( List<UUID> apps ) throws Exception {
-
- for ( UUID app : apps ) {
-
- final EntityManager em;
- try {
- em = getEmf().getEntityManager( app );
- } catch (Throwable ex) {
- log.error("Error getting Entity Manager, aborting", ex);
- return;
- }
-
- query(em, "product_productid = 'B006K2ZZ7K'") ;
- query(em, "review_profilename = 'Twoapennything'") ;
- query(em, "review_profilename contains 'Natalia'") ;
- query(em, "review_profilename contains 'Patrick'") ;
- query(em, "review_time = 1342051200") ;
- query(em, "review_time > 1342051200") ;
- query(em, "review_score > 0");
- query(em, "review_score > 2");
- query(em, "review_score > 3");
- query(em, "review_score > 4");
- query(em, "review_score > 5");
- }
- }
-
- public static void query( EntityManager em, String query ) throws Exception {
- Query q = Query.fromQL(query) ;
- Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
- log.info("size = {} returned from query {}",results.size(), q.getQl() );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3e0bf8d/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
new file mode 100644
index 0000000..a818199
--- /dev/null
+++ b/stack/corepersistence/perftest1/src/test/java/org/apache/usergrid/persistence/Usergrid1PerformanceTest.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.persistence;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.commons.lang.NumberUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * TODO: make configurable, add CHOP markup.
+ */
+public class Usegrid1PerformanceTest {
+ private static final Logger log = LoggerFactory.getLogger(Usegrid1PerformanceTest.class);
+
+ // max entities we will write and read
+ static int maxEntities = Integer.MAX_VALUE;
+
+ // each app will get all data
+ static int orgCount = 2;
+ static int appCount = 5 ;
+
+ // number of threads = orgCount x appCount
+
+ // total number of records = orgCount x appCount x numRecords
+
+ private final EntityManagerFactory emf;
+
+
+ public Usegrid1PerformanceTest() throws Throwable {
+ emf = UsergridBootstrap.newInstance().getBean( EntityManagerFactory.class );
+ }
+
+
+ public EntityManagerFactory getEmf() {
+ return emf;
+ }
+
+
+ @Test
+ public void loadAndReadData() throws Exception {
+
+ log.info("Start Data Load");
+ List<UUID> apps = loadData();
+ log.info("Finish Data Load");
+
+ log.info("Start Data Read");
+ readData( apps );
+ log.info("Finish Data Read");
+
+ runSelectedQueries( apps );
+
+ }
+
+
+ private List<UUID> loadData() throws Exception {
+
+ long time = new Date().getTime();
+
+ List<UUID> apps = new ArrayList<UUID>();
+ List<Thread> threads = new ArrayList<Thread>();
+
+ for ( int i=0; i<orgCount; i++ ) {
+
+ for ( int j=0; j<appCount; j++ ) {
+
+ UUID appId = getEmf().createApplication(
+ "testorg-" + RandomStringUtils.randomAlphanumeric(6),
+ "testapp-" + RandomStringUtils.randomAlphanumeric(6));
+
+ apps.add( appId );
+
+ Thread t = new Thread( new DataLoader( appId ));
+ t.start();
+ threads.add(t);
+ }
+ }
+
+ // wait for indexing to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+
+ return apps;
+ }
+
+
+ private void readData( List<UUID> apps ) throws InterruptedException {
+
+ List<Thread> threads = new ArrayList<Thread>();
+ for ( UUID app : apps ) {
+
+ Thread t = new Thread( new DataReader( app ));
+ t.start();
+ threads.add(t);
+ }
+
+ // wait for reading to end
+ for ( Thread t : threads ) {
+ t.join();
+ }
+ }
+
+
+ private class DataReader implements Runnable {
+ UUID app;
+
+ public DataReader( UUID app ) {
+ this.app = app;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ UUID appId = app;
+
+ Query query = Query.fromQL( "review_score > 0"); // get all reviews;
+ query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
+
+ Results results;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ return;
+ }
+
+ results.getEntities(); // cause retrieval from Cassandra
+ int count = results.size();
+
+ while ( results.hasCursor() && count < maxEntities ) {
+ query.setCursor( results.getCursor() ) ;
+ try {
+ results = em.searchCollection( em.getApplicationRef(), "reviews", query );
+ } catch (Exception ex) {
+ log.error("Error on search, aborting", ex);
+ log.info("Read {} reviews in {}", count, appId );
+ return;
+ }
+ results.getEntities(); // cause retrieval from Cassanda;
+ count += results.size();
+
+ log.info("Read {} reviews in {}", count, appId );
+ }
+ }
+ }
+
+
+ private class DataLoader implements Runnable {
+ UUID app;
+
+ public DataLoader( UUID scope ) {
+ this.app = scope;
+ }
+
+ public void run() {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ BufferedReader br;
+ try {
+ InputStreamReader isr = new InputStreamReader(
+ getClass().getResourceAsStream("/finefoods.txt")); // TODO: make configurable
+ br = new BufferedReader(isr);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error opening file", ex);
+ }
+ String s = null;
+
+ // create the first entry
+ Map<String, Object> currentEntityMap = new HashMap<String, Object>();
+
+ UUID appId = app;
+
+ int count = 0;
+ try {
+ while ( (s = br.readLine()) != null && count < maxEntities ) {
+
+ try {
+
+ if ( s.trim().equals("")) { // then we are at end of a record
+
+ // write and index current entity
+ Entity entity = em.create("review", currentEntityMap );
+
+ if ( maxEntities < 20 ) {
+ log.info("Index written for {}", entity.getUuid());
+ log.info("---");
+ }
+
+ // create the next entity
+ currentEntityMap = new HashMap<String, Object>();
+
+ count++;
+ if (count % 100000 == 0) {
+ log.info("Indexed {} reviews in {}", count, appId );
+ }
+ continue;
+ }
+
+ // process a field
+ String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
+ String value = s.substring( s.indexOf(":") + 1 ).trim();
+
+ if ( maxEntities < 20 ) {
+ log.info("Indexing {} = {}", name, value);
+ }
+
+ if ( NumberUtils.isNumber(value) && value.contains(".")) {
+ currentEntityMap.put( name, Double.parseDouble(value));
+
+ } else if ( NumberUtils.isNumber(value) ) {
+ currentEntityMap.put( name, Long.parseLong(value));
+
+ } else {
+ currentEntityMap.put( name, value.toString() );
+ }
+
+ } catch ( Exception e ) {
+ log.info("Error on line " + count);
+ }
+ }
+
+ } catch (IOException ex) {
+ throw new RuntimeException("Error reading file", ex);
+ }
+ }
+ }
+
+
+ public void runSelectedQueries( List<UUID> apps ) throws Exception {
+
+ for ( UUID app : apps ) {
+
+ final EntityManager em;
+ try {
+ em = getEmf().getEntityManager( app );
+ } catch (Throwable ex) {
+ log.error("Error getting Entity Manager, aborting", ex);
+ return;
+ }
+
+ query(em, "product_productid = 'B006K2ZZ7K'") ;
+ query(em, "review_profilename = 'Twoapennything'") ;
+ query(em, "review_profilename contains 'Natalia'") ;
+ query(em, "review_profilename contains 'Patrick'") ;
+ query(em, "review_time = 1342051200") ;
+ query(em, "review_time > 1342051200") ;
+ query(em, "review_score > 0");
+ query(em, "review_score > 2");
+ query(em, "review_score > 3");
+ query(em, "review_score > 4");
+ query(em, "review_score > 5");
+ }
+ }
+
+ public static void query( EntityManager em, String query ) throws Exception {
+ Query q = Query.fromQL(query) ;
+ Results results = em.searchCollection( em.getApplicationRef(), "reviews", q );
+ log.info("size = {} returned from query {}",results.size(), q.getQl() );
+ }
+
+}