You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/03/19 17:26:47 UTC
[1/2] incubator-usergrid git commit: WIP Overwrite
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-486 [created] 12c2a1a06
WIP Overwrite
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9a787238
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9a787238
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9a787238
Branch: refs/heads/USERGRID-486
Commit: 9a7872380b2953b3916197d2aacc65fa78867642
Parents: eb0c689
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 18 14:18:14 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 18 14:18:14 2015 -0600
----------------------------------------------------------------------
.../persistence/index/guice/IndexTestFig.java | 57 ++++
.../index/guice/TestIndexModule.java | 4 +
.../index/impl/CorePerformanceIT.java | 339 -------------------
.../index/impl/IndexLoadTestsIT.java | 169 +++++++++
4 files changed, 230 insertions(+), 339 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9a787238/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/IndexTestFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/IndexTestFig.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/IndexTestFig.java
new file mode 100644
index 0000000..ecf3dfa
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/IndexTestFig.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.index.guice;
+
+
+import java.util.UUID;
+
+import org.safehaus.guicyfig.Default;
+import org.safehaus.guicyfig.FigSingleton;
+import org.safehaus.guicyfig.GuicyFig;
+import org.safehaus.guicyfig.Key;
+
+
+/**
+ * Test configuration for creating documents
+ */
+@FigSingleton
+public interface IndexTestFig extends GuicyFig {
+
+ @Key( "stresstest.numWorkers" )
+ @Default( "16" )
+ public int getNumberOfWorkers();
+
+ @Key( "stresstest.numberofRecords" )
+ @Default( "10000" )
+ public int getNumberOfRecords();
+
+ @Key( "stresstest.bufferSize" )
+ @Default( "1000" )
+ public int getBufferSize();
+
+ @Key( "stresstest.validate.wait" )
+ @Default( "2000" )
+ public long getValidateWait();
+
+
+ @Key( "stresstest.applicationId" )
+ @Default( "0df46683-cdab-11e4-83c2-d2be4de3081a" )
+ public String getApplicationId();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9a787238/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 50b994d..79a021a 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@ -19,9 +19,12 @@
package org.apache.usergrid.persistence.index.guice;
+import org.safehaus.guicyfig.GuicyFigModule;
+
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.TestModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.impl.BufferQueue;
import org.apache.usergrid.persistence.index.impl.BufferQueueInMemoryImpl;
import org.apache.usergrid.persistence.index.impl.BufferQueueSQSImpl;
@@ -36,5 +39,6 @@ public class TestIndexModule extends TestModule {
// configure collections and our core astyanax framework
install( new CollectionModule() );
install( new IndexModule() );
+ install( new GuicyFigModule(IndexTestFig.class) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9a787238/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
deleted file mode 100644
index c1bfe38..0000000
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
+++ /dev/null
@@ -1,339 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence.index.impl;
-
-
-import java.io.BufferedReader;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import org.junit.ClassRule;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang3.math.NumberUtils;
-
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.SearchTypes;
-import org.apache.usergrid.persistence.index.guice.TestIndexModule;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-import org.apache.usergrid.persistence.index.query.EntityResults;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.field.DoubleField;
-import org.apache.usergrid.persistence.model.field.LongField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-
-
-
-/**
- * TODO: make CorePerformanceIT configurable, add CHOP markup.
- */
-public class CorePerformanceIT extends BaseIT {
- private static final Logger log = LoggerFactory.getLogger(CorePerformanceIT.class);
-
- @ClassRule
- public static ElasticSearchResource es = new ElasticSearchResource();
-
- // max entities we will write and read
- static int maxEntities = 10; // TODO: make this configurable when you add Chop
-
- // each app will get all data
- static int appCount = 10;
-
- // number of threads = orgCount x appCount
-
- // total number of records = orgCount x appCount x numRecords
-
- static EntityCollectionManagerFactory ecmf;
- static EntityIndexFactory ecif ;
-
-
- @Ignore("Relies on finefoods.txt which must be downloaded separately")
- @Test
- public void loadAndReadData() throws IOException, InterruptedException {
-
- Injector injector = Guice.createInjector( new TestIndexModule() );
-
- // only on first run
- //MigrationManager m = injector.getInstance( MigrationManager.class )
- //m.migrate()
-
- ecmf = injector.getInstance( EntityCollectionManagerFactory.class );
- ecif = injector.getInstance( EntityIndexFactory.class );
-
- final ApplicationScope scope = new ApplicationScopeImpl( new SimpleId( "application" ) );
-
- log.info("Start Data Load");
-
- List<IndexScope> scopes = loadData(scope);
-
- log.info("Finish Data Load");
-
- log.info("Start Data Read");
-
-
- readData( scope, scopes );
- log.info("Finish Data Read");
-
- runSelectedQueries( scope, scopes );
-
- }
-
-
- private List<IndexScope> loadData(final ApplicationScope applicationScope) throws InterruptedException {
-
- long time = new Date().getTime();
-
- List<IndexScope> scopes = new ArrayList<IndexScope>();
- List<Thread> threads = new ArrayList<Thread>();
-
-
- for ( int j = 0; j < appCount; j++ ) {
-
- String appName = "app-" + j + "-" + time;
- Id appId = new SimpleId( appName );
- IndexScope indexScope = new IndexScopeImpl( appId, "reviews");
- scopes.add( indexScope );
-
- Thread t = new Thread( new DataLoader( applicationScope, indexScope ) );
- t.start();
- threads.add( t );
- }
-
- // wait for indexing to end
- for ( Thread t : threads ) {
- t.join();
- }
-
- return scopes;
- }
-
-
- private void readData(final ApplicationScope applicationScope, List<IndexScope> scopes ) throws InterruptedException {
-
- List<Thread> threads = new ArrayList<Thread>();
- for ( IndexScope scope : scopes ) {
-
- Thread t = new Thread( new DataReader( applicationScope, scope ));
- t.start();
- threads.add(t);
- }
-
- // wait for reading to end
- for ( Thread t : threads ) {
- t.join();
- }
- }
-
-
- static class DataReader implements Runnable {
- final ApplicationScope scope;
- final IndexScope indexScope;
-
- public DataReader( final ApplicationScope scope, IndexScope indexScope ) {
- this.scope = scope;
- this.indexScope = indexScope;
- }
-
- public void run() {
-
- EntityIndex eci = ecif.createEntityIndex( scope);
- EntityCollectionManager ecm = ecmf.createCollectionManager( new CollectionScopeImpl( scope.getApplication(), indexScope.getOwner(), indexScope.getName() ) );
-
- Query query = Query.fromQL( "review_score > 0"); // get all reviews;
- query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
-
- final SearchTypes searchType = SearchTypes.fromTypes( "review" );
-
- CandidateResults candidateResults = eci.search(indexScope, searchType, query );
- int count = candidateResults.size();
-
- while ( candidateResults.hasCursor() && count < maxEntities ) {
- query.setCursor( candidateResults.getCursor() ) ;
- candidateResults = eci.search(indexScope, searchType, query );
- count += candidateResults.size();
-
- //cause retrieval from cassandra
- EntityResults entityResults = new EntityResults(
- candidateResults, ecm, UUIDGenerator.newTimeUUID() );
-
- while(entityResults.hasNext()){
- entityResults.next();
- }
-
- log.info("Read {} reviews in {} / {} ", new Object[] {
- count, indexScope.getOwner(), indexScope.getName() } );
- }
- }
- }
-
-
- static class DataLoader implements Runnable {
- final ApplicationScope applicationScope;
- final IndexScope indexScope;
-
- public DataLoader( final ApplicationScope applicationScope, IndexScope indexScope ) {
- this.applicationScope = applicationScope;
- this.indexScope = indexScope;
- }
-
- public void run() {
-
- CollectionScope collectionScope = new CollectionScopeImpl(
- applicationScope.getApplication(), indexScope.getOwner(), indexScope.getName() );
- EntityCollectionManager ecm = ecmf.createCollectionManager(collectionScope );
- EntityIndex eci = ecif.createEntityIndex(applicationScope );
-
- FileReader fr;
- try {
- fr = new FileReader("../../resources/finefoods.txt");
- } catch (FileNotFoundException ex) {
- throw new RuntimeException("Error opening file", ex);
- }
- BufferedReader br = new BufferedReader(fr);
- String s = null;
-
- // create the first entry
- Entity current = new Entity(
- new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
-
-// Id orgId = orgAppScope.scope.getApplication();
-// Id appId = orgAppScope.scope.getOwner();
-
- int count = 0;
-
- EntityIndexBatch entityIndexBatch = eci.createBatch();
-
- try {
- while ( (s = br.readLine()) != null && count < maxEntities ) {
-
- try {
-
- if ( s.trim().equals("")) { // then we are at end of a record
-
- // write and index current entity
- ecm.write( current ).toBlocking().last();
-
- entityIndexBatch.index(indexScope, current );
-
- if ( maxEntities < 20 ) {
- log.info("Index written for {}", current.getId());
- log.info("---");
- }
-
- // create the next entity
- current = new Entity(
- new SimpleId(UUIDGenerator.newTimeUUID(), "review"));
-
- count++;
- if(count % 1000 == 0){
- entityIndexBatch.execute().get();
- }
-
- if (count % 100000 == 0) {
- log.info("Indexed {} reviews in {} / {} ",
- new Object[] {
- count,
- applicationScope,
- indexScope.getOwner() } );
- }
- continue;
- }
-
- // process a field
- String name = s.substring( 0, s.indexOf(":")).replace("/", "_").toLowerCase() ;
- String value = s.substring( s.indexOf(":") + 1 ).trim();
-
- if ( maxEntities < 20 ) {
- log.info("Indexing {} = {}", name, value);
- }
-
- if ( NumberUtils.isNumber(value) && value.contains(".")) {
- current.setField( new DoubleField( name, Double.parseDouble(value)));
-
- } else if ( NumberUtils.isNumber(value) ) {
- current.setField( new LongField( name, Long.parseLong(value)));
-
- } else {
- current.setField( new StringField( name, value.toString() ));
- }
-
- } catch ( Exception e ) {
- log.info("Error on line " + count);
- }
- }
-
- } catch (IOException ex) {
- throw new RuntimeException("Error reading file", ex);
- }
-
- eci.refresh();
- }
- }
-
-
- public void runSelectedQueries(final ApplicationScope scope, List<IndexScope> indexScopes ) {
-
- for ( IndexScope indexScope : indexScopes ) {
- EntityIndex eci = ecif.createEntityIndex(scope );
-
- // TODO: come up with more and more complex queries for CorePerformanceIT
-
- query(indexScope, eci, "product_productid = 'B006K2ZZ7K'") ;
- query(indexScope, eci, "review_profilename = 'Twoapennything'") ;
- query(indexScope, eci, "review_profilename contains 'Natalia'") ;
- query(indexScope, eci, "review_profilename contains 'Patrick'") ;
- query(indexScope, eci, "review_time = 1342051200") ;
- query(indexScope, eci, "review_time > 1342051200") ;
- query(indexScope, eci, "review_score > 0");
- query(indexScope, eci, "review_score > 2");
- query(indexScope, eci, "review_score > 3");
- query(indexScope, eci, "review_score > 4");
- query(indexScope, eci, "review_score > 5");
- }
- }
-
- public static void query(final IndexScope indexScope, final EntityIndex eci, final String query ) {;
- Query q = Query.fromQL(query) ;
-// CandidateResults candidateResults = eci.search(indexScope, q ); TODO FIXME
-// log.info("size = {} returned from query {}", candidateResults.size(), q.getQl() );
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9a787238/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
new file mode 100644
index 0000000..c962d6b
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+
+import java.util.List;
+import java.util.UUID;
+
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.guice.IndexTestFig;
+import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.IntegerField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.google.inject.Inject;
+
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Action2;
+import rx.functions.Func1;
+import rx.schedulers.Schedulers;
+
+
+/**
+ * TODO: make CorePerformanceIT configurable, add CHOP markup.
+ */
+@RunWith( EsRunner.class )
+@UseModules( { TestIndexModule.class } )
+public class IndexLoadTestsIT extends BaseIT {
+ private static final Logger log = LoggerFactory.getLogger( IndexLoadTestsIT.class );
+
+ @ClassRule
+ public static ElasticSearchResource es = new ElasticSearchResource();
+
+
+ @Inject
+ public IndexTestFig indexTestFig;
+
+ @Inject
+ public EntityIndexFactory entityIndexFactory;
+
+ @Test
+ public void testHeavyLoad(){
+
+ final UUID applicationUUID = UUID.fromString( indexTestFig.getApplicationId() );
+
+ final Id applicationId = new SimpleId(applicationUUID, "application");
+ final ApplicationScope scope = new ApplicationScopeImpl( applicationId );
+
+ final EntityIndex index = entityIndexFactory.createEntityIndex( scope );
+
+ //create our index if it doesn't exist
+ index.initializeIndex();
+
+ final Observable<Entity> createEntities = createStreamFromWorkers( index, applicationId );
+
+ //run them all
+ createEntities.toBlocking().last();
+
+
+
+
+ }
+
+ public Observable<Entity> createStreamFromWorkers(final EntityIndex entityIndex, final Id ownerId){
+
+ //create a sequence of observables. Each index will be it's own worker thread using the Schedulers.newthread()
+ return Observable.range( 0, indexTestFig.getNumberOfWorkers() ).parallel( new Func1<Observable<Integer>, Observable<Entity>>() {
+
+
+ @Override
+ public Observable<Entity> call( final Observable<Integer> integerObservable ) {
+ return integerObservable.flatMap( new Func1<Integer, Observable<Entity>>() {
+ @Override
+ public Observable<Entity> call( final Integer integer ) {
+ return createWriteObservable( entityIndex, ownerId, integer );
+ }
+ } );
+
+ }
+ }, Schedulers.newThread() );
+ }
+
+
+ private Observable<Entity> createWriteObservable( final EntityIndex entityIndex, final Id ownerId, final int workerIndex){
+
+
+ final IndexScope scope = new IndexScopeImpl( ownerId, "test" );
+
+
+
+ return Observable.range( 0, indexTestFig.getNumberOfRecords() )
+
+ //create our entity
+ .map( new Func1<Integer, Entity>() {
+ @Override
+ public Entity call( final Integer integer ) {
+ final Entity entity = new Entity("test");
+
+ entity.setField( new IntegerField("workerIndex", workerIndex));
+ entity.setField( new IntegerField( "ordinal", integer ) );
+
+ return entity;
+ }
+ } ).buffer( indexTestFig.getBufferSize() ).doOnNext( new Action1<List<Entity>>() {
+ @Override
+ public void call( final List<Entity> entities ) {
+ //take our entities and roll them into a batch
+ Observable.from( entities ).collect( entityIndex.createBatch(), new Action2<EntityIndexBatch, Entity>() {
+
+
+ @Override
+ public void call( final EntityIndexBatch entityIndexBatch, final Entity entity ) {
+ entityIndexBatch.index(scope, entity );
+ }
+ } ).doOnNext( new Action1<EntityIndexBatch>() {
+ @Override
+ public void call( final EntityIndexBatch entityIndexBatch ) {
+ entityIndexBatch.execute();
+ }
+ } ).toBlocking().last();
+ }
+ } )
+
+ //translate back into a stream of entities for the caller to use
+ .flatMap( new Func1<List<Entity>, Observable<Entity>>() {
+ @Override
+ public Observable<Entity> call( final List<Entity> entities ) {
+ return Observable.from( entities );
+ }
+ } );
+
+ }
+
+
+
+
+}
[2/2] incubator-usergrid git commit: Merge branch 'USERGRID-405' into
USERGRID-486
Posted by to...@apache.org.
Merge branch 'USERGRID-405' into USERGRID-486
Conflicts:
stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/12c2a1a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/12c2a1a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/12c2a1a0
Branch: refs/heads/USERGRID-486
Commit: 12c2a1a063da04f674eb654f5116a898a5ddd43d
Parents: 9a78723 8ea46ba
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 19 10:26:42 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 19 10:26:42 2015 -0600
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 6 +-
stack/core/pom.xml | 10 +
.../AllApplicationsObservable.java | 39 ++
.../usergrid/corepersistence/CoreModule.java | 88 ++-
.../corepersistence/CpEntityManagerFactory.java | 109 ++-
.../corepersistence/CpManagerCache.java | 55 +-
.../corepersistence/CpRelationManager.java | 7 +-
.../usergrid/corepersistence/ManagerCache.java | 12 +-
.../events/EntityVersionDeletedHandler.java | 16 +-
.../migration/CoreDataVersions.java | 43 ++
.../migration/CoreMigration.java | 53 ++
.../migration/CoreMigrationPlugin.java | 64 ++
.../migration/EntityDataMigration.java | 147 ----
.../migration/EntityTypeMappingMigration.java | 78 ++-
.../migration/GraphShardVersionMigration.java | 152 -----
.../migration/MigrationModuleVersionPlugin.java | 133 ++++
.../migration/MigrationSystemVersions.java | 43 ++
.../corepersistence/migration/Versions.java | 14 -
.../results/FilteringLoader.java | 50 +-
.../rx/AllEntitiesInSystemObservable.java | 101 ---
.../rx/ApplicationObservable.java | 128 ----
.../rx/EdgesFromSourceObservable.java | 63 --
.../rx/EdgesToTargetObservable.java | 63 --
.../corepersistence/rx/TargetIdObservable.java | 66 --
.../rx/impl/AbstractGraphVisitorImpl.java | 107 +++
.../rx/impl/AllApplicationsObservableImpl.java | 142 ++++
.../rx/impl/AllEntitiesInSystemImpl.java | 62 ++
.../rx/impl/AllNodesInGraphImpl.java | 55 ++
.../corepersistence/util/CpEntityMapUtils.java | 19 +-
.../corepersistence/util/CpNamingUtils.java | 2 +
.../persistence/EntityManagerFactory.java | 20 -
.../cassandra/EntityManagerFactoryImpl.java | 23 -
.../corepersistence/StaleIndexCleanupTest.java | 48 ++
.../migration/EntityDataMigrationIT.java | 262 -------
.../migration/EntityTypeMappingMigrationIT.java | 183 ++---
.../migration/GraphShardVersionMigrationIT.java | 226 ------
.../MigrationModuleVersionPluginTest.java | 259 +++++++
.../migration/MigrationTestRule.java | 99 ---
.../migration/TestProgressObserver.java | 71 --
.../rx/AllEntitiesInSystemObservableIT.java | 41 +-
.../rx/ApplicationObservableTestIT.java | 14 +-
.../rx/EdgesFromSourceObservableIT.java | 10 +-
.../rx/EdgesToTargetObservableIT.java | 8 +-
.../rx/TargetIdObservableTestIT.java | 7 +-
.../org/apache/usergrid/persistence/GeoIT.java | 5 +-
.../usergrid/persistence/LargeEntityIT.java | 3 +-
.../collection/EntityCollectionManager.java | 6 -
.../EntityCollectionManagerFactory.java | 27 +-
.../collection/EntityCollectionManagerSync.java | 49 --
.../persistence/collection/MvccEntity.java | 5 +-
.../persistence/collection/MvccLogEntry.java | 1 +
.../cache/CachedEntityCollectionManager.java | 7 -
.../collection/event/EntityVersionDeleted.java | 3 +-
.../collection/guice/CollectionModule.java | 45 +-
.../persistence/collection/guice/Write.java | 17 -
.../collection/guice/WriteUpdate.java | 17 -
.../EntityCollectionManagerFactoryImpl.java | 71 +-
.../impl/EntityCollectionManagerImpl.java | 119 +---
.../impl/EntityCollectionManagerSyncImpl.java | 71 --
.../collection/impl/EntityDeletedTask.java | 8 +-
.../impl/EntityVersionCleanupTask.java | 191 +++---
.../mvcc/MvccEntitySerializationStrategy.java | 102 ---
.../mvcc/stage/delete/MarkCommit.java | 2 +-
.../mvcc/stage/write/WriteCommit.java | 9 +-
.../collection/mvcc/stage/write/WriteStart.java | 6 +-
.../collection/serialization/EntityRepair.java | 38 --
.../MvccEntitySerializationStrategy.java | 119 ++++
.../serialization/OptimisticUpdate.java | 23 -
.../UniqueValueSerializationStrategy.java | 46 +-
.../serialization/UniqueValueSet.java | 4 +
.../impl/CollectionDataVersions.java | 44 ++
.../serialization/impl/EntityRepairImpl.java | 149 ----
.../impl/EntityVersionSerializer.java | 2 -
.../serialization/impl/FieldSerializer.java | 118 ----
.../MvccEntitySerializationStrategyImpl.java | 24 +-
...vccEntitySerializationStrategyProxyImpl.java | 151 ++--
.../MvccEntitySerializationStrategyV1Impl.java | 9 +-
.../MvccEntitySerializationStrategyV2Impl.java | 8 +-
.../MvccEntitySerializationStrategyV3Impl.java | 615 +++++++++++++++++
.../serialization/impl/SerializationModule.java | 85 ++-
.../serialization/impl/UniqueFieldEntry.java | 61 ++
.../impl/UniqueFieldEntrySerializer.java | 141 ++++
.../impl/UniqueFieldRowKeySerializer.java | 122 ++++
.../UniqueValueSerializationStrategyImpl.java | 254 +++++--
.../impl/migration/CollectionMigration.java | 53 ++
.../migration/CollectionMigrationPlugin.java | 68 ++
.../impl/migration/EntityIdScope.java | 49 ++
.../migration/MvccEntityDataMigrationImpl.java | 283 ++++++++
.../collection/util/EntityUtils.java | 19 +-
.../collection/EntityCollectionManagerIT.java | 104 +--
.../EntityCollectionManagerSyncIT.java | 193 ------
.../collection/guice/TestCollectionModule.java | 22 +-
.../impl/EntityVersionCleanupTaskTest.java | 680 ++++++++-----------
.../impl/EntityVersionCreatedTaskTest.java | 2 -
.../mvcc/stage/delete/MarkCommitTest.java | 2 +-
.../mvcc/stage/write/FieldSerializerTest.java | 52 --
.../write/UniqueFieldRowKeySerializerTest.java | 52 ++
...niqueValueSerializationStrategyImplTest.java | 149 +++-
.../mvcc/stage/write/WriteCommitTest.java | 2 +-
.../mvcc/stage/write/WriteStartTest.java | 6 +-
.../serialization/EntityRepairImplTest.java | 147 ----
.../impl/LogEntryIteratorTest.java | 3 +-
...MvccEntitySerializationStrategyImplTest.java | 117 +---
...cEntitySerializationStrategyProxyV1Test.java | 85 ---
...ntitySerializationStrategyProxyV1_3Test.java | 83 +++
...cEntitySerializationStrategyProxyV2Test.java | 83 ---
...ntitySerializationStrategyProxyV2_3Test.java | 84 +++
...ccEntitySerializationStrategyV1ImplTest.java | 80 ++-
...ccEntitySerializationStrategyV2ImplTest.java | 75 +-
.../MvccEntitySerializationStrategyV2Test.java | 7 +-
...ccEntitySerializationStrategyV3ImplTest.java | 74 ++
.../impl/UniqueFieldEntrySerializerTest.java | 108 +++
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 215 ++++++
.../MvccEntityDataMigrationV1ToV3ImplTest.java | 105 +++
.../MvccEntityDataMigrationV2ToV3ImplTest.java | 99 +++
.../collection/util/LogEntryMock.java | 39 +-
.../collection/util/UniqueValueEntryMock.java | 161 +++++
.../collection/util/VersionGenerator.java | 55 ++
.../usergrid/persistence/core/CPManager.java | 28 +
.../astyanax/DynamicCompositeParserImpl.java | 81 +++
.../core/astyanax/FieldBufferBuilder.java | 15 +
.../core/astyanax/FieldBufferParser.java | 13 +
.../core/astyanax/MultiRowColumnIterator.java | 12 +-
.../persistence/core/guice/CommonModule.java | 17 +-
.../persistence/core/guice/CurrentImpl.java | 42 --
.../persistence/core/guice/PreviousImpl.java | 42 --
.../core/hystrix/HystrixCassandra.java | 94 ---
.../migration/data/AbstractMigrationPlugin.java | 128 ++++
.../core/migration/data/DataMigration.java | 97 +--
.../migration/data/DataMigrationManager.java | 18 +-
.../data/DataMigrationManagerImpl.java | 234 ++++---
.../migration/data/MigrationDataProvider.java | 43 ++
.../core/migration/data/MigrationInfoCache.java | 52 ++
.../migration/data/MigrationInfoCacheImpl.java | 84 +++
.../data/MigrationInfoSerialization.java | 28 +-
.../data/MigrationInfoSerializationImpl.java | 67 +-
.../core/migration/data/MigrationPlugin.java | 56 ++
.../migration/data/MigrationRelationship.java | 100 +++
.../core/migration/data/PluginPhase.java | 42 ++
.../core/migration/data/ProgressObserver.java | 63 ++
.../core/migration/data/VersionedData.java | 38 ++
.../migration/data/VersionedMigrationSet.java | 153 +++++
.../core/scope/ApplicationScope.java | 1 +
.../persistence/core/task/TaskExecutor.java | 4 +-
.../core/guice/DataMigrationResetRule.java | 88 +++
.../core/guice/MaxMigrationModule.java | 39 --
.../core/guice/MaxMigrationVersion.java | 40 --
.../core/guice/MigrationManagerRule.java | 7 +
.../core/guice/TestCommonModule.java | 1 +
.../persistence/core/guice/TestModule.java | 5 +-
.../data/DataMigrationManagerImplTest.java | 350 ++++++----
.../data/MigrationInfoSerializationTest.java | 32 +-
.../data/TestMigrationDataProvider.java | 61 ++
.../migration/data/TestProgressObserver.java | 89 +++
.../data/VersionedMigrationSetTest.java | 198 ++++++
.../persistence/core/util/IdGenerator.java | 51 ++
.../persistence/graph/GraphManager.java | 3 +-
.../persistence/graph/GraphManagerFactory.java | 2 +
.../persistence/graph/guice/GraphModule.java | 103 ++-
.../graph/impl/GraphManagerImpl.java | 32 +-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 9 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 16 +-
.../impl/stage/NodeDeleteListenerImpl.java | 16 +-
.../EdgeMetadataSerialization.java | 3 +-
.../graph/serialization/EdgesObservable.java | 33 +
.../graph/serialization/TargetIdObservable.java | 38 ++
.../EdgeMetadataSerializationProxyImpl.java | 161 +++--
.../impl/EdgeMetadataSerializationV1Impl.java | 6 +
.../impl/EdgeMetadataSerializationV2Impl.java | 6 +
.../serialization/impl/EdgesObservableImpl.java | 80 +++
.../serialization/impl/GraphDataVersions.java | 43 ++
.../impl/GraphManagerFactoryImpl.java | 95 +++
.../impl/NodeSerializationImpl.java | 36 +-
.../impl/TargetIdObservableImpl.java | 72 ++
.../impl/migration/EdgeDataMigrationImpl.java | 149 ++++
.../impl/migration/GraphMigration.java | 53 ++
.../impl/migration/GraphMigrationPlugin.java | 69 ++
.../serialization/impl/migration/GraphNode.java | 39 ++
.../shard/count/NodeShardApproximationImpl.java | 4 +-
.../NodeShardCounterSerializationImpl.java | 25 +-
.../shard/impl/NodeShardAllocationImpl.java | 16 +-
.../shard/impl/ShardGroupCompactionImpl.java | 64 +-
.../persistence/graph/GraphManagerIT.java | 21 +-
.../persistence/graph/GraphManagerLoadTest.java | 13 +-
.../graph/GraphManagerShardConsistencyIT.java | 10 +-
.../graph/GraphManagerShardingIT.java | 13 +-
.../graph/GraphManagerStressTest.java | 11 +-
.../graph/guice/TestGraphModule.java | 27 +-
.../graph/impl/EdgeDeleteListenerTest.java | 11 +-
.../graph/impl/NodeDeleteListenerTest.java | 9 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 7 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 29 +-
.../EdgeMetaDataSerializationProxyV1Test.java | 14 +-
.../EdgeMetaDataSerializationProxyV2Test.java | 21 +-
.../EdgeMetaDataSerializationV1Test.java | 7 +-
.../EdgeMetaDataSerializationV2Test.java | 7 +-
.../EdgeMetadataSerializationTest.java | 39 +-
.../EdgeSerializationChopTest.java | 7 +-
.../serialization/EdgeSerializationTest.java | 21 +-
.../serialization/NodeSerializationTest.java | 15 +-
.../migration/EdgeDataMigrationImplTest.java | 177 +++++
.../impl/shard/EdgeShardSerializationTest.java | 7 +-
.../impl/shard/NodeShardAllocationTest.java | 27 +-
.../impl/shard/NodeShardCacheTest.java | 7 +-
.../impl/shard/ShardGroupCompactionTest.java | 7 +-
.../shard/count/NodeShardApproximationTest.java | 9 +-
.../NodeShardCounterSerializationTest.java | 6 +-
.../shard/impl/ShardEntryGroupIteratorTest.java | 15 +-
...rceDirectedEdgeDescendingComparatorTest.java | 23 +-
...getDirectedEdgeDescendingComparatorTest.java | 23 +-
.../graph/test/util/EdgeTestUtils.java | 29 +-
.../usergrid/persistence/map/MapManager.java | 25 +-
.../persistence/map/guice/MapModule.java | 8 +-
.../map/impl/MapManagerFactoryImpl.java | 29 +-
.../persistence/map/guice/TestMapModule.java | 3 +-
.../persistence/model/entity/Entity.java | 23 +-
.../persistence/model/entity/EntityMap.java | 66 ++
.../model/entity/EntityToMapConverter.java | 133 ++++
.../model/entity/MapToEntityConverter.java | 203 ++++++
.../persistence/model/entity/SimpleId.java | 1 -
.../persistence/model/field/AbstractField.java | 8 +
.../persistence/model/field/ListField.java | 2 +-
.../persistence/model/field/StringField.java | 20 +
.../usergrid/persistence/index/EntityIndex.java | 4 +-
.../index/impl/EsEntityIndexImpl.java | 96 +--
.../index/guice/TestIndexModule.java | 6 +-
.../index/impl/EntityIndexMapUtils.java | 52 +-
.../queue/guice/TestQueueModule.java | 3 +-
stack/pom.xml | 2 +-
.../org/apache/usergrid/rest/IndexResource.java | 3 +
.../apache/usergrid/rest/MigrateResource.java | 94 ++-
.../applications/ApplicationsResource.java | 6 -
232 files changed, 9242 insertions(+), 5174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12c2a1a0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
index 79a021a,27fe705..23dfe06
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/guice/TestIndexModule.java
@@@ -37,8 -32,6 +34,7 @@@ public class TestIndexModule extends Te
install( new CommonModule());
// configure collections and our core astyanax framework
- install( new CollectionModule() );
- install( new IndexModule() );
+ install( new IndexModule() );
+ install( new GuicyFigModule(IndexTestFig.class) );
}
}