You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/04/02 04:20:52 UTC
[3/3] git commit: Updated the property configurations so that network
strategy and simple strategies are interchangeable
Updated the property configurations so that network strategy and simple strategies are interchangeable
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/064d2ae0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/064d2ae0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/064d2ae0
Branch: refs/heads/asyncqueue
Commit: 064d2ae098d156e086b1a321bf95d721c63e42fb
Parents: 289d66c
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Apr 1 18:41:46 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Apr 1 18:41:46 2014 -0700
----------------------------------------------------------------------
.gitignore | 3 +-
.../migration/MigrationManagerFig.java | 3 -
.../migration/MigrationManagerImpl.java | 30 +--
.../src/test/resources/usergrid-UNIT.properties | 5 +-
.../persistence/graph/GraphManagerLoadTest.java | 264 +++++++++++++++++++
.../src/test/resources/usergrid-AWS.properties | 17 ++
.../src/test/resources/usergrid-UNIT.properties | 5 +-
7 files changed, 301 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/064d2ae0/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index c9f4468..8da5658 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,4 +35,5 @@ stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/q
/stack/corepersistence/perftest2/nbactions.xml
/stack/corepersistence/perftest1/nbactions.xml
/stack/corepersistence/perftest1/src/main/resources/usergrid.properties
-/stack/corepersistence/perftest2/src/main/resources/usergrid.properties
\ No newline at end of file
+/stack/corepersistence/perftest2/src/main/resources/usergrid.properties
+aws.properties
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/064d2ae0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerFig.java
index 235df1a..1caa656 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerFig.java
@@ -19,7 +19,4 @@ public interface MigrationManagerFig extends GuicyFig {
@Key( "collections.keyspace.strategy.options" )
String getStrategyOptions();
- @Key( "collections.keyspace.replicationfactor" )
- @Default( "1" )
- String getReplicationFactor();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/064d2ae0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerImpl.java
index 47b4ac5..fc051de 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/migration/MigrationManagerImpl.java
@@ -36,17 +36,15 @@ public class MigrationManagerImpl implements MigrationManager {
private final Set<Migration> migrations;
private final Keyspace keyspace;
- private final Properties props;
private final MigrationManagerFig fig;
@Inject
- public MigrationManagerImpl( final Keyspace keyspace, final Set<Migration> migrations, final Properties props,
+ public MigrationManagerImpl( final Keyspace keyspace, final Set<Migration> migrations,
MigrationManagerFig fig ) {
this.keyspace = keyspace;
this.migrations = migrations;
- this.props = props;
this.fig = fig;
}
@@ -133,10 +131,7 @@ public class MigrationManagerImpl implements MigrationManager {
}
- ImmutableMap.Builder<String, Object> strategyOptions =
- ImmutableMap.<String, Object>builder().put( "replication_factor", fig.getReplicationFactor() );
-
- strategyOptions.putAll( getKeySpaceProps() );
+ ImmutableMap.Builder<String, Object> strategyOptions = getKeySpaceProps();
ImmutableMap<String, Object> options =
@@ -153,19 +148,22 @@ public class MigrationManagerImpl implements MigrationManager {
/**
* Get keyspace properties
*/
- private Map<String, String> getKeySpaceProps() {
- Map<String, String> keyspaceProps = new HashMap<String, String>();
+ private ImmutableMap.Builder<String, Object> getKeySpaceProps() {
+ ImmutableMap.Builder<String, Object> keyspaceProps = ImmutableMap.<String, Object>builder();
- for ( Map.Entry<Object, Object> entry : props.entrySet() ) {
- final String key = entry.getKey().toString();
+ String optionString = fig.getStrategyOptions();
- if ( ! key.startsWith( fig.getKeyByMethod( "getStrategyOptions" ) ) ) {
- continue;
- }
+ if(optionString == null){
+ return keyspaceProps;
+ }
+
+
+
+ for ( String key : optionString.split( "," ) ) {
- final String optionKey = key.substring( fig.getKeyByMethod( "getStrategyOptions" ).length() + 1 );
+ final String[] options = key.split( ":" );
- keyspaceProps.put( optionKey, entry.getValue().toString() );
+ keyspaceProps.put( options[0], options[1] );
}
return keyspaceProps;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/064d2ae0/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
index 152693a..8680e98 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
@@ -8,7 +8,6 @@ collections.keyspace=Usergrid_Collections
cassandra.timeout=5000
-collections.keyspace.replicationfactor=1
-collections.keyspace.strategy.options=
+collections.keyspace.strategy.options=replication_factor:3
+collections.keyspace.strategy.class=SimpleStrategy
collection.stage.transient.timeout=60
-collections.keyspace.strategy.class=org.apache.cassandra.locator.SimpleStrategy
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/064d2ae0/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
new file mode 100644
index 0000000..721de17
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.graph;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.jukito.JukitoRunner;
+import org.jukito.UseModules;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.time.StopWatch;
+
+import org.apache.usergrid.persistence.collection.OrganizationScope;
+import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.collection.impl.OrganizationScopeImpl;
+import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Inject;
+
+import rx.Observable;
+import rx.Subscriber;
+
+import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
+import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+
+@RunWith( JukitoRunner.class )
+@UseModules( TestGraphModule.class )
+public class GraphManagerLoadTest {
+ private static final Logger log = LoggerFactory.getLogger( GraphManagerLoadTest.class );
+
+ @Inject
+ private GraphManagerFactory factory;
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+
+ protected OrganizationScope scope;
+
+ protected int numWorkers;
+ protected int writeLimit;
+ protected int readCount;
+
+
+ @Before
+ public void setupOrg() {
+ //get the system property of the UUID to use. If one is not set, use the defualt
+ String uuidString = System.getProperty( "org.id", "80a42760-b699-11e3-a5e2-0800200c9a66" );
+
+ scope = new OrganizationScopeImpl( createId( UUID.fromString( uuidString ), "test" ) );
+
+ numWorkers = Integer.parseInt( System.getProperty( "numWorkers", "100" ) );
+ writeLimit = Integer.parseInt( System.getProperty( "writeLimit", "10000" ) );
+ readCount = Integer.parseInt( System.getProperty( "readCount", "20000" ) );
+ }
+
+
+// @Ignore
+ @Test
+ public void writeThousandsSingleSource() throws InterruptedException, ExecutionException {
+ EdgeGenerator generator = new EdgeGenerator() {
+
+ private Id sourceId = createId( "source" );
+
+
+ @Override
+ public Edge newEdge() {
+ Edge edge = createEdge( sourceId, "test", createId( "target" ) );
+
+
+ return edge;
+ }
+
+
+ @Override
+ public Observable<Edge> doSearch( final GraphManager manager ) {
+ UUID uuid = UUIDGenerator.newTimeUUID();
+
+ return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", uuid, null ) );
+ }
+ };
+
+ doTest( generator );
+ }
+
+
+ @Test
+ @Ignore
+ public void writeThousandsSingleTarget() throws InterruptedException, ExecutionException {
+ EdgeGenerator generator = new EdgeGenerator() {
+
+ private Id targetId = createId( "target" );
+
+
+ @Override
+ public Edge newEdge() {
+ Edge edge = createEdge( createId( "source" ), "test", targetId );
+
+
+ return edge;
+ }
+
+
+ @Override
+ public Observable<Edge> doSearch( final GraphManager manager ) {
+ UUID uuid = UUIDGenerator.newTimeUUID();
+
+ return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", uuid, null ) );
+ }
+ };
+
+ doTest( generator );
+ }
+
+
+ /**
+ * Execute the test with the generator
+ */
+ private void doTest( EdgeGenerator generator ) throws InterruptedException, ExecutionException {
+
+ ExecutorService executor = Executors.newFixedThreadPool( numWorkers );
+
+ List<Future<Boolean>> futures = new ArrayList<Future<Boolean>>( numWorkers );
+
+ for ( int i = 0; i < numWorkers; i++ ) {
+ Future<Boolean> future = executor.submit( new Worker( generator, writeLimit, readCount ) );
+
+ futures.add( future );
+ }
+
+
+ /**
+ * Block on all workers until they're done
+ */
+ for ( Future<Boolean> future : futures ) {
+ future.get();
+ }
+ }
+
+
+ private class Worker implements Callable<Boolean> {
+ private final EdgeGenerator generator;
+ private final int writeLimit;
+ private final int readCount;
+
+
+ private Worker( final EdgeGenerator generator, final int writeLimit, final int readCount ) {
+ this.generator = generator;
+ this.writeLimit = writeLimit;
+ this.readCount = readCount;
+ }
+
+
+ @Override
+ public Boolean call() throws Exception {
+ GraphManager manager = factory.createEdgeManager( scope );
+
+
+ final StopWatch timer = new StopWatch();
+ timer.start();
+
+ for ( int i = 0; i < writeLimit; i++ ) {
+
+ Edge edge = generator.newEdge();
+
+ Edge returned = manager.writeEdge( edge ).toBlockingObservable().last();
+
+
+ assertNotNull( "Returned has a version", returned.getVersion() );
+
+
+ if ( i % 1000 == 0 ) {
+ log.info( " Wrote: " + i );
+ }
+ }
+
+ timer.stop();
+ log.info( "Total time to write {} entries {} ms", writeLimit, timer.getTime() );
+ timer.reset();
+
+ timer.start();
+
+ final CountDownLatch latch = new CountDownLatch( 1 );
+
+
+ generator.doSearch( manager ).take( readCount ).subscribe( new Subscriber<Edge>() {
+ @Override
+ public void onCompleted() {
+ timer.stop();
+ latch.countDown();
+ }
+
+
+ @Override
+ public void onError( final Throwable throwable ) {
+ fail( "Exception occurced " + throwable );
+ }
+
+
+ @Override
+ public void onNext( final Edge edge ) {
+
+ }
+ } );
+
+
+ latch.await();
+
+
+ log.info( "Total time to read {} entries {} ms", readCount, timer.getTime() );
+
+ return true;
+ }
+ }
+
+
+ private interface EdgeGenerator {
+
+ /**
+ * Create a new edge to persiste
+ */
+ public Edge newEdge();
+
+ public Observable<Edge> doSearch( final GraphManager manager );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/064d2ae0/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
new file mode 100644
index 0000000..078e414
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
@@ -0,0 +1,17 @@
+# Keep nothing but overriding test defaults in here
+cassandra.connections=100
+cassandra.port=9160
+cassandra.version=1.2
+#cassandra.hosts=ec2-54-211-120-157.compute-1.amazonaws.com, ec2-54-235-236-189.compute-1.amazonaws.com, ec2-107-22-110-71.compute-1.amazonaws.com, ec2-54-242-241-70.compute-1.amazonaws.com, ec2-54-205-71-218.compute-1.amazonaws.com, ec2-54-205-70-94.compute-1.amazonaws.com
+cassandra.hosts=localhost
+cassandra.cluster_name=Usergrid
+collections.keyspace=Usergrid_Collections
+cassandra.timeout=5000
+
+
+#collections.keyspace.strategy.options=us-east-1:3
+#collections.keyspace.strategy.class=NetworkTopologyStrategy
+collections.keyspace.strategy.options=replication_factor:3
+collections.keyspace.strategy.class=SimpleStrategy
+collection.stage.transient.timeout=60
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/064d2ae0/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
index 152693a..8680e98 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
@@ -8,7 +8,6 @@ collections.keyspace=Usergrid_Collections
cassandra.timeout=5000
-collections.keyspace.replicationfactor=1
-collections.keyspace.strategy.options=
+collections.keyspace.strategy.options=replication_factor:3
+collections.keyspace.strategy.class=SimpleStrategy
collection.stage.transient.timeout=60
-collections.keyspace.strategy.class=org.apache.cassandra.locator.SimpleStrategy