You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2014/12/22 16:36:00 UTC
[01/50] incubator-usergrid git commit: Cleaned up extra lines.
Repository: incubator-usergrid
Updated Branches:
refs/heads/UG-rest-test-framework-overhaul 615ac49bc -> 81edb677f
Cleaned up extra lines.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/67f74225
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/67f74225
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/67f74225
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 67f742256eb36a97e612f3e6b6ca9b5a7026be93
Parents: 0537e10
Author: grey <gr...@apigee.com>
Authored: Wed Oct 29 14:01:51 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Wed Oct 29 14:01:51 2014 -0700
----------------------------------------------------------------------
.../events/EntityVersionCreatedImpl.java | 3 ---
.../persistence/collection/guice/CollectionModule.java | 13 -------------
.../collection/impl/EntityCollectionManagerImpl.java | 11 -----------
.../persistence/index/impl/EsEntityIndexBatchImpl.java | 4 ----
4 files changed, 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/67f74225/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedImpl.java
index 738183c..c7aada7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedImpl.java
@@ -39,9 +39,6 @@ public class EntityVersionCreatedImpl implements EntityVersionCreated{
HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
-// CpEntityManagerFactory emf = (CpEntityManagerFactory)
-// CpSetup.getInjector().getInstance( EntityManagerFactory.class );
-
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
EntityIndexBatch batch = ei.createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/67f74225/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index de99078..4c96303 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@ -93,19 +93,6 @@ public class CollectionModule extends AbstractModule {
}
-
- @Provides
- public List<EntityVersionDeleted> emptyEntityVersionDeletedInitialization(){
- return Collections.EMPTY_LIST;
-
- }
-
- @Provides
- public List<EntityDeleted> emptyEntityDeletedInitialization(){
- return Collections.EMPTY_LIST;
-
- }
-
@Provides
@Singleton
@Inject
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/67f74225/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 3d33a52..5b6265c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -159,7 +159,6 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
ValidationUtils.verifyIdentity(entityId);
-
// create our observable and start the write
final CollectionIoEvent<Entity> writeData =
new CollectionIoEvent<Entity>(collectionScope, entity);
@@ -174,20 +173,10 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
observable.map(writeCommit).doOnNext(new Action1<Entity>() {
@Override
public void call(final Entity entity) {
-//<<<<<<< Updated upstream
-//
-// // TODO fire a task here
-//
-// taskExecutor.submit(entityVersionCleanupFactory.getTask(
-// collectionScope, entityId, entity.getVersion() ));
-//
-// // post-processing to come later. leave it empty for now.
-//=======
//TODO fire the created task first then the entityVersioncleanup
taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope,entity));
taskExecutor.submit(entityVersionCleanupFactory.getTask(collectionScope, entityId,entity.getVersion()));
//post-processing to come later. leave it empty for now.
-//>>>>>>> Stashed changes
}
}).doOnError(rollback);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/67f74225/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 1fca845..ca7f666 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -222,10 +222,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
( STRING_PREFIX + ENTITYID_FIELDNAME,entity.getId().getUuid().toString().toLowerCase() ),
FilterBuilders.rangeFilter("version").lt( entity.getId().getUuid().timestamp() ));
-
-
- //QueryBuilders.rangeQuery( )
-
DeleteByQueryResponse response = client.prepareDeleteByQuery("test")
.setQuery( fqb ).execute().actionGet();
[16/50] incubator-usergrid git commit: Pruning old code and
reformatting
Posted by sf...@apache.org.
Pruning old code and reformatting
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/54a64e6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/54a64e6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/54a64e6d
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 54a64e6d0cdf21a2f0199c266a3fe8a512442ccb
Parents: 2c3875c
Author: grey <gr...@apigee.com>
Authored: Mon Nov 3 09:52:59 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Mon Nov 3 09:52:59 2014 -0800
----------------------------------------------------------------------
.../usergrid/corepersistence/GuiceModule.java | 2 --
.../events/EntityVersionCreatedHandler.java | 14 -----------
.../impl/EntityCollectionManagerImpl.java | 9 -------
.../impl/EntityVersionCreatedTaskTest.java | 5 ----
.../index/impl/EsEntityIndexBatchImpl.java | 25 +++++++++-----------
5 files changed, 11 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54a64e6d/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index e723a35..3b868b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@ -64,8 +64,6 @@ public class GuiceModule extends AbstractModule {
= Multibinder.newSetBinder( binder(),EntityVersionCreated.class );
versionCreatedMultibinder.addBinding().to(EntityVersionCreatedHandler.class);
- bind( Boolean.class ).annotatedWith( Names.named( "defaultConfig" ) ).toInstance( false );
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54a64e6d/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 6f3c507..ff09119 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -45,10 +45,6 @@ import com.google.inject.name.Named;
*/
public class EntityVersionCreatedHandler implements EntityVersionCreated {
- @Inject
- @Named( "defaultConfig" )
- private Boolean defaultConfig;
-
private static final Logger logger = LoggerFactory.getLogger(EntityVersionCreatedHandler.class );
public EntityVersionCreatedHandler() {
@@ -69,16 +65,6 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
EntityIndexBatch batch = ei.createBatch();
- //loads test properties, if not found then remove stale entities
-// String[] locations = { "usergrid-properties-context.xml" };
-// ConfigurableApplicationContext appContext =
-// new ClassPathXmlApplicationContext( locations );
-//
-// Properties properties = (Properties)appContext.getBean("properties");
-// String staleString = properties.getProperty("allow.stale.entities","false");
-
- //if(!staleString.contains( "true" ))
-
if(System.getProperty( "allow.stale.entities","false" ).equals( "false" )) {
batch.deindexPreviousVersions( entity );
batch.execute();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54a64e6d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 323b838..16046b3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -288,29 +288,20 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final Id entityId = entity.getId();
-
ValidationUtils.verifyIdentity(entityId);
// create our observable and start the write
CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>(collectionScope, entity);
-
Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner(writeData, writeUpdate);
-
return observable.map(writeCommit).doOnNext(new Action1<Entity>() {
@Override
public void call(final Entity entity) {
logger.debug("sending entity to the queue");
//we an update, signal the fix
-
taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope,entity));
- //taskExecutor.submit(entityVersionCleanupFactory.getTask(collectionScope, entityId,entity.getVersion()));
- //TODO T.N Change this to fire a task
- // Observable.from( new CollectionIoEvent<Id>(collectionScope,
- // entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
-
}
}).doOnError(rollback);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54a64e6d/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 3326856..b25b5b4 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -58,12 +58,7 @@ public class EntityVersionCreatedTaskTest {
final int sizeToReturn = 0;
- final CountDownLatch latch = new CountDownLatch( sizeToReturn );
-
- final EntityVersionCreatedTest eventListener = new EntityVersionCreatedTest(latch);
-
final Set<EntityVersionCreated> listeners = mock( Set.class );
- final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
when ( listeners.size()).thenReturn( 0 );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54a64e6d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index e8ee96f..7f6cf70 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -222,25 +222,22 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
return this;
}
+
@Override
- public EntityIndexBatch deindexPreviousVersions(Entity entity){
+ public EntityIndexBatch deindexPreviousVersions( Entity entity ) {
- //String helper = System.getProperty( "allow.stale.entities","false" );
- //"false" gets returned if there is no allow.stale.entities process
- FilteredQueryBuilder fqb = QueryBuilders.filteredQuery( QueryBuilders
- .termQuery( STRING_PREFIX + ENTITYID_FIELDNAME,
- entity.getId().getUuid().toString().toLowerCase() ),
- FilterBuilders.rangeFilter( ENTITYVERSION_FIELDNAME ).lt( entity.getVersion().timestamp() ) );
+ FilteredQueryBuilder fqb = QueryBuilders.filteredQuery( QueryBuilders
+ .termQuery( STRING_PREFIX + ENTITYID_FIELDNAME,
+ entity.getId().getUuid().toString().toLowerCase() ),
+ FilterBuilders.rangeFilter( ENTITYVERSION_FIELDNAME ).lt( entity.getVersion().timestamp() ) );
- DeleteByQueryResponse response =
- client.prepareDeleteByQuery( indexName ).setQuery( fqb ).execute().actionGet();
+ DeleteByQueryResponse response = client.prepareDeleteByQuery( indexName ).setQuery( fqb ).execute().actionGet();
- //error message needs to be retooled so that it describes the entity more througly
- logger.debug( "Deleted entity {}:{} from all index scopes with response status = {}",
- new Object[] { entity.getId().getType(), entity.getId().getUuid(), response.status().toString() } );
+ //error message needs to be retooled so that it describes the entity more throughly
+ logger.debug( "Deleted entity {}:{} from all index scopes with response status = {}",
+ new Object[] { entity.getId().getType(), entity.getId().getUuid(), response.status().toString() } );
- maybeFlush();
- // }
+ maybeFlush();
return this;
}
[40/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/21c93f1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/21c93f1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/21c93f1e
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 21c93f1e30c23cc6f24dd50bf49271098ae680fa
Parents: 88d49cb 277eb73
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Dec 1 09:38:37 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Dec 1 09:38:37 2014 -0500
----------------------------------------------------------------------
.../rest/management/users/UsersResource.java | 38 ++++++++++++--------
1 file changed, 24 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[46/50] incubator-usergrid git commit: Add missing headers and
formatting only.
Posted by sf...@apache.org.
Add missing headers and formatting only.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/afe52d47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/afe52d47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/afe52d47
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: afe52d47f3c0f549ecf6db2a9fe5c0d0053c8af7
Parents: 7f1d533
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Dec 10 10:45:58 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Dec 10 10:45:58 2014 -0500
----------------------------------------------------------------------
.../core/task/NamedTaskExecutorImpl.java | 47 +++++++++++++-------
.../core/task/NamedTaskExecutorImplTest.java | 24 +++++++---
2 files changed, 50 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/afe52d47/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
index a022c08..9007167 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.usergrid.persistence.core.task;
@@ -9,8 +27,6 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.ForkJoinPool;
-import java.util.concurrent.ForkJoinWorkerThread;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.RejectedExecutionHandler;
@@ -60,15 +76,16 @@ public class NamedTaskExecutorImpl implements TaskExecutor {
this.name = name;
this.poolSize = poolSize;
- //The user has chosen to disable asynchronous execution, to create an executor service that will reject all requests
- if(poolSize == 0){
+ // The user has chosen to disable asynchronous execution,
+ // to create an executor service that will reject all requests
+ if ( poolSize == 0 ) {
executorService = MoreExecutors.listeningDecorator( new RejectingExecutorService());
}
//queue executions as normal
else {
- final BlockingQueue<Runnable> queue =
- queueLength > 0 ? new ArrayBlockingQueue<Runnable>( queueLength ) : new SynchronousQueue<Runnable>();
+ final BlockingQueue<Runnable> queue = queueLength > 0
+ ? new ArrayBlockingQueue<Runnable>(queueLength) : new SynchronousQueue<Runnable>();
executorService = MoreExecutors.listeningDecorator( new MaxSizeThreadPool( queue ) );
}
@@ -83,9 +100,7 @@ public class NamedTaskExecutorImpl implements TaskExecutor {
try {
future = executorService.submit( task );
- /**
- * Log our success or failures for debugging purposes
- */
+ // Log our success or failures for debugging purposes
Futures.addCallback( future, new TaskFutureCallBack<V>( task ) );
}
catch ( RejectedExecutionException ree ) {
@@ -210,7 +225,8 @@ public class NamedTaskExecutorImpl implements TaskExecutor {
@Override
- public boolean awaitTermination( final long timeout, final TimeUnit unit ) throws InterruptedException {
+ public boolean awaitTermination( final long timeout, final TimeUnit unit )
+ throws InterruptedException {
return false;
}
@@ -241,22 +257,23 @@ public class NamedTaskExecutorImpl implements TaskExecutor {
@Override
- public <T> List<Future<T>> invokeAll( final Collection<? extends Callable<T>> tasks, final long timeout,
- final TimeUnit unit ) throws InterruptedException {
+ public <T> List<Future<T>> invokeAll( final Collection<? extends Callable<T>> tasks,
+ final long timeout, final TimeUnit unit ) throws InterruptedException {
+
throw new RejectedExecutionException("No Asynchronous tasks allowed");
}
@Override
public <T> T invokeAny( final Collection<? extends Callable<T>> tasks )
- throws InterruptedException, ExecutionException {
+ throws InterruptedException, ExecutionException {
throw new RejectedExecutionException("No Asynchronous tasks allowed");
}
@Override
- public <T> T invokeAny( final Collection<? extends Callable<T>> tasks, final long timeout, final TimeUnit unit )
- throws InterruptedException, ExecutionException, TimeoutException {
+ public <T> T invokeAny( final Collection<? extends Callable<T>> tasks, final long timeout,
+ final TimeUnit unit ) throws InterruptedException, ExecutionException, TimeoutException {
throw new RejectedExecutionException("No Asynchronous tasks allowed");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/afe52d47/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
index 65189f1..4f95918 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
@@ -1,16 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.usergrid.persistence.core.task;
-
import java.util.ArrayList;
import java.util.List;
-import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
-
import org.junit.Test;
-
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
[03/50] incubator-usergrid git commit: Revert "Created extra step to
help mock out the test."
Posted by sf...@apache.org.
Revert "Created extra step to help mock out the test."
This reverts commit e6760f2569a9b1fb10ba5155120d59ecf5f4b733.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/839fc2b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/839fc2b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/839fc2b5
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 839fc2b5d62212ce3cffc4de47addba51ba9b9fd
Parents: e6760f2
Author: grey <gr...@apigee.com>
Authored: Wed Oct 29 16:30:59 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Wed Oct 29 16:30:59 2014 -0700
----------------------------------------------------------------------
.../persistence/collection/impl/EntityVersionCreatedTask.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/839fc2b5/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
index cef4248..b78b09c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
@@ -122,9 +122,7 @@ public class EntityVersionCreatedTask implements Task<Void> {
}
if ( listenerSize == 1 ) {
- EntityVersionCreated entityVersionCreated = listeners.iterator().next();
-
- entityVersionCreated.versionCreated( collectionScope,entity );
+ listeners.iterator().next().versionCreated( collectionScope,entity );
return;
}
[36/50] incubator-usergrid git commit: Anther merge fix.
Posted by sf...@apache.org.
Anther merge fix.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1884cce1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1884cce1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1884cce1
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 1884cce15e299f31c72caa1ae15cc07bf9545dc0
Parents: 2ee0eff
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Nov 24 10:42:55 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Nov 24 10:42:55 2014 -0500
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/GuiceModule.java | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1884cce1/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index 98f473a..a57158f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@ -65,6 +65,13 @@ public class GuiceModule extends AbstractModule {
install(new MapModule());
install(new QueueModule());
+ bind(ManagerCache.class).to( CpManagerCache.class );
+
+ Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
+ dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
+ dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
+ //dataMigrationMultibinder.addBinding().to( EntityDataMigration.class );
+
Multibinder<EntityDeleted> entityBinder =
Multibinder.newSetBinder(binder(), EntityDeleted.class);
entityBinder.addBinding().to(EntityDeletedHandler.class);
@@ -77,10 +84,7 @@ public class GuiceModule extends AbstractModule {
Multibinder.newSetBinder( binder(), EntityVersionCreated.class );
versionCreatedMultibinder.addBinding().to(EntityVersionCreatedHandler.class);
- Multibinder<DataMigration> dataMigrationMultibinder =
- Multibinder.newSetBinder( binder(), DataMigration.class );
- dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
- dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
+
}
}
[24/50] incubator-usergrid git commit: Merge branch
'two-dot-o-events' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o-events' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-events
# By Dave Johnson
# Via Dave Johnson
* 'two-dot-o-events' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid:
Fixes to StaleEntityCleapupTest, which is looking good now, also some import cleanups.
test fixes
Conflicts:
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/216e0505
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/216e0505
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/216e0505
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 216e0505b0b3d1b52c18043554f8b365ae2c38ee
Parents: 3294f00 3d3bbed
Author: grey <gr...@apigee.com>
Authored: Tue Nov 4 15:52:43 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Nov 4 15:52:43 2014 -0800
----------------------------------------------------------------------
.../usergrid/corepersistence/GuiceModule.java | 1 -
.../events/EntityVersionCreatedHandler.java | 9 +----
.../corepersistence/StaleIndexCleanupTest.java | 37 ++++++++------------
.../corepersistence/TestGuiceModule.java | 2 --
.../collection/EntityCollectionManager.java | 1 -
.../impl/EntityVersionCreatedTaskTest.java | 21 -----------
.../index/impl/EsEntityIndexBatchImpl.java | 5 ---
7 files changed, 15 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/216e0505/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 525c3a2,266b0f9..e47d460
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@@ -21,7 -21,8 +21,6 @@@ package org.apache.usergrid.persistence
import java.util.Collection;
import org.apache.usergrid.persistence.core.util.Health;
-
-import org.apache.usergrid.persistence.collection.event.EntityDeleted;
-
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/216e0505/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 3ba8fe1,92bb3ce..90055a4
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@@ -17,19 -17,11 +17,10 @@@
*/
package org.apache.usergrid.persistence.collection.impl;
- import com.google.common.base.Optional;
--import com.google.common.util.concurrent.ListenableFuture;
- import com.netflix.astyanax.Keyspace;
- import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
- import java.util.ArrayList;
- import java.util.HashSet;
import java.util.Iterator;
- import java.util.List;
- import java.util.NoSuchElementException;
import java.util.Set;
- import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import org.apache.usergrid.persistence.collection.CollectionScope;
@@@ -47,16 -31,11 +30,12 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
- import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Test;
- import static org.mockito.Matchers.any;
- import static org.mockito.Matchers.same;
import static org.mockito.Mockito.mock;
- import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
[47/50] incubator-usergrid git commit: Remove write-delay from
iterating query tests and, instead,
decrease the number of threads used to process tasks.
Posted by sf...@apache.org.
Remove write-delay from iterating query tests and, instead, decrease the number of threads used to process tasks.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a290e0d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a290e0d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a290e0d6
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: a290e0d6b9d72befef6b50fcb313a5c6d4055557
Parents: afe52d4
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Dec 10 14:03:11 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Dec 10 14:03:11 2014 -0500
----------------------------------------------------------------------
stack/core/.test-tdoe-5thread.log.swp | Bin 0 -> 28672 bytes
.../corepersistence/CpRelationManager.java | 2 --
.../migration/GraphShardVersionMigrationIT.java | 5 ++---
.../query/AbstractIteratingQueryIT.java | 2 +-
.../test/resources/usergrid-custom-test.properties | 3 +++
5 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a290e0d6/stack/core/.test-tdoe-5thread.log.swp
----------------------------------------------------------------------
diff --git a/stack/core/.test-tdoe-5thread.log.swp b/stack/core/.test-tdoe-5thread.log.swp
new file mode 100644
index 0000000..dccd541
Binary files /dev/null and b/stack/core/.test-tdoe-5thread.log.swp differ
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a290e0d6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 135688e..5039a41 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -60,8 +60,6 @@ import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a290e0d6/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
index 3ab9f63..f287047 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/migration/GraphShardVersionMigrationIT.java
@@ -28,7 +28,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.apache.usergrid.AbstractCoreIT;
-import org.apache.usergrid.CoreApplication;
import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.corepersistence.EntityWriteHelper;
import org.apache.usergrid.corepersistence.ManagerCache;
@@ -37,11 +36,9 @@ import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManagerImpl;
import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
@@ -52,6 +49,7 @@ import rx.functions.Action1;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import org.junit.Ignore;
public class GraphShardVersionMigrationIT extends AbstractCoreIT {
@@ -83,6 +81,7 @@ public class GraphShardVersionMigrationIT extends AbstractCoreIT {
@Test
+ @Ignore("Ignored awaiting fix for USERGRID-268")
public void testIdMapping() throws Throwable {
assertEquals( "version 2 expected", 2, graphShardVersionMigration.getVersion() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a290e0d6/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
index d8f31ff..e4eb1c5 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
@@ -7,7 +7,7 @@
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
- *
+*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a290e0d6/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index a1ff5b2..d093ca8 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -20,3 +20,6 @@ cassandra.timeout=2000
cassandra.connections=1000
hystrix.threadpool.graph_user.coreSize=50
hystrix.threadpool.graph_async.coreSize=50
+
+collection.task.pool.threadsize=8
+collection.task.pool.queuesize=8
[31/50] incubator-usergrid git commit: Moved delete by query into
index, since they cannot be batched
Posted by sf...@apache.org.
Moved delete by query into index, since they cannot be batched
Fixed version to be reserved field and updated mapping
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/43105f12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/43105f12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/43105f12
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 43105f126dbe17b4446870e7760260829ef92ad9
Parents: 73a3bcc
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 18 17:19:19 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 18 17:19:19 2014 -0700
----------------------------------------------------------------------
.../events/EntityDeletedHandler.java | 5 +-
.../events/EntityVersionCreatedHandler.java | 1 +
.../usergrid/persistence/index/EntityIndex.java | 21 ++++-
.../persistence/index/EntityIndexBatch.java | 13 ----
.../index/impl/EsEntityIndexBatchImpl.java | 66 ++++------------
.../index/impl/EsEntityIndexImpl.java | 81 ++++++++++++++++++--
.../persistence/index/impl/IndexingUtils.java | 19 ++++-
7 files changed, 127 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/43105f12/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index b5cb1ff..bd738fe 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -55,9 +55,6 @@ public class EntityDeletedHandler implements EntityDeleted {
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
- EntityIndexBatch batch = ei.createBatch();
-
- batch.deleteEntity( entityId );
- batch.execute();
+ ei.deleteAllVersionsOfEntity( entityId );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/43105f12/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 94a673a..08b8fbd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -58,6 +58,7 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
EntityIndexBatch batch = ei.createBatch();
+ //TODO why aren't we using a collection fig here? This seems kludgy
if ( System.getProperty( "allow.stale.entities", "false" ).equals( "false" )) {
batch.deindexPreviousVersions( entity );
batch.execute();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/43105f12/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 88498b3..29d8015 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -19,9 +19,12 @@
package org.apache.usergrid.persistence.index;
+import java.util.UUID;
+
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -48,8 +51,24 @@ public interface EntityIndex {
/**
* Get the candidate results of all versions of the entity for this id.
+ * @param indexScope The scope of the index to search in
+ * @param id The id to search within.
+ */
+ public CandidateResults getEntityVersions(final IndexScope indexScope, final Id id);
+
+ /**
+ * Create a delete method that deletes by Id. This will delete all documents from ES with the same entity Id,
+ * effectively removing all versions of an entity from all index scopes
+ * @param entityId The entityId to remove
+ */
+ public void deleteAllVersionsOfEntity(final Id entityId );
+
+ /**
+ * Takes all the previous versions of the current entity and deletes all previous versions
+ * @param id The id to remove
+ * @param version The max version to retain
*/
- public CandidateResults getEntityVersions(final IndexScope indexScope, Id id);
+ public void deletePreviousVersions(final Id id, final UUID version);
/**
* Refresh the index.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/43105f12/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
index bd1ee40..68008bf 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -56,19 +56,6 @@ public interface EntityIndexBatch {
*/
public EntityIndexBatch deindex(final IndexScope scope, final Id id, final UUID version);
- /**
- * Create a delete method that deletes by Id. This will delete all documents from ES with
- * the same entity Id, effectively removing all versions of an entity from all index scopes.
- */
- public EntityIndexBatch deleteEntity( Id entityId );
-
- /**
- * Takes all the previous versions of the current entity and deindexs all previous versions
- * @param entity
- * @return
- */
- public EntityIndexBatch deindexPreviousVersions(Entity entity);
-
/**
* Execute the batch
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/43105f12/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index f69a64c..e0ec1ed 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -29,7 +29,14 @@ import java.util.UUID;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest;
+import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
+import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.client.Client;
+import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.FilteredQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermQueryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,18 +64,17 @@ import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
-import com.google.common.base.Joiner;
-
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -167,7 +173,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
- log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
+ log.debug( "De-indexing type {} with documentId '{}'", entityType, indexId );
bulkRequest.add( client.prepareDelete( indexName, entityType, indexId ).setRefresh( refresh ) );
@@ -192,43 +198,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
return deindex( indexScope, entity.getId(), entity.getVersion() );
}
-
- @Override
- public EntityIndexBatch deleteEntity(Id entityId) {
-
- TermQueryBuilder tqb = QueryBuilders.termQuery(
- STRING_PREFIX + ENTITYID_FIELDNAME, entityId.getUuid().toString().toLowerCase());
-
- DeleteByQueryResponse response = client.prepareDeleteByQuery( indexName )
- .setQuery( tqb ).execute().actionGet();
-
- logger.debug("Deleted entity {}:{} from all index scopes with response status = {}",
- new Object[] { entityId.getType(), entityId.getUuid(), response.status().toString() });
-
- maybeFlush();
-
- return this;
- }
-
-
- @Override
- public EntityIndexBatch deindexPreviousVersions( Entity entity ) {
-
- FilteredQueryBuilder fqb = QueryBuilders.filteredQuery( QueryBuilders
- .termQuery( STRING_PREFIX + ENTITYID_FIELDNAME,
- entity.getId().getUuid().toString().toLowerCase() ),
- FilterBuilders.rangeFilter( ENTITYVERSION_FIELDNAME ).lt( entity.getVersion().timestamp() ) );
-
- DeleteByQueryResponse response = client.prepareDeleteByQuery( indexName ).setQuery( fqb ).execute().actionGet();
-
- //error message needs to be retooled so that it describes the entity more throughly
- logger.debug( "Deleted entity {}:{} from all index scopes with response status = {}",
- new Object[] { entity.getId().getType(), entity.getId().getUuid(), response.status().toString() } );
-
- maybeFlush();
-
- return this;
- }
@Override
@@ -251,9 +220,6 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
try {
responses = request.execute().actionGet();
- } catch (Throwable t) {
- logger.error("Unable to communicate with elasticsearch");
- failureMonitor.fail("Unable to execute batch", t);
}
catch ( Throwable t ) {
log.error( "Unable to communicate with elasticsearch" );
@@ -293,9 +259,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
/**
* Set the entity as a map with the context
+ *
* @param entity The entity
* @param context The context this entity appears in
- * @return
*/
private static Map entityToMap( final Entity entity, final String context ) {
final Map entityMap = entityToMap( entity );
@@ -304,8 +270,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
entityMap.put( ENTITY_CONTEXT_FIELDNAME, context );
//but the fieldname
- //we have to prefix because we use query equality to seek this later. TODO see if we can make this more declarative
- entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() );
+ //we have to prefix because we use query equality to seek this later. TODO see if we can make this more
+ // declarative
+ entityMap.put( ENTITY_ID_FIELDNAME, IndexingUtils.idString( entity.getId() ).toLowerCase() );
+
+ entityMap.put( ENTITY_ID_FIELDNAME, IndexingUtils.idString( entity.getId() ).toLowerCase() );
return entityMap;
}
@@ -428,5 +397,4 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private void initBatch() {
this.bulkRequest = client.prepareBulk();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/43105f12/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index b1e5374..406c967 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
@@ -31,6 +32,8 @@ import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
+import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
+import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
@@ -40,11 +43,15 @@ import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.FilteredQueryBuilder;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -65,6 +72,7 @@ import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -74,7 +82,8 @@ import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
@@ -221,11 +230,9 @@ public class EsEntityIndexImpl implements EntityIndex {
xcb ) // set mapping as the default for all types
.execute().actionGet();
- if(!pitr.isAcknowledged()){
+ if ( !pitr.isAcknowledged() ) {
throw new IndexException( "Unable to create default mappings" );
}
-
-
}
@@ -251,7 +258,6 @@ public class EsEntityIndexImpl implements EntityIndex {
.setScroll( cursorTimeout + "m" ).setQuery( qb );
-
final FilterBuilder fb = query.createFilterBuilder();
@@ -300,7 +306,6 @@ public class EsEntityIndexImpl implements EntityIndex {
}
-
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
this.indexName, context, entityTypes, srb
@@ -412,7 +417,9 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
public int getPendingTasks() {
- final PendingClusterTasksResponse tasksResponse = esProvider.getClient().admin().cluster().pendingClusterTasks( new PendingClusterTasksRequest() ).actionGet();
+ final PendingClusterTasksResponse tasksResponse =
+ esProvider.getClient().admin().cluster().pendingClusterTasks( new PendingClusterTasksRequest() )
+ .actionGet();
return tasksResponse.pendingTasks().size();
}
@@ -447,7 +454,65 @@ public class EsEntityIndexImpl implements EntityIndex {
failureMonitor.success();
- return parseResults( searchResponse, new Query( ) );
+ return parseResults( searchResponse, new Query() );
+ }
+
+
+ @Override
+ public void deleteAllVersionsOfEntity( Id entityId ) {
+
+ final TermQueryBuilder tqb =
+ QueryBuilders.termQuery( ENTITY_ID_FIELDNAME, entityId.getUuid().toString().toLowerCase() );
+
+
+ final DeleteByQueryResponse response =
+ esProvider.getClient().prepareDeleteByQuery( indexName ).setQuery( tqb ).execute().actionGet();
+
+
+ logger.debug( "Deleted entity {}:{} from all index scopes with response status = {}",
+ new Object[] { entityId.getType(), entityId.getUuid(), response.status().toString() } );
+
+ checkDeleteByQueryResponse( tqb, response );
+
+ }
+
+
+ @Override
+ public void deletePreviousVersions( final Id id, final UUID version ) {
+
+ final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
+ QueryBuilders.termQuery( ENTITY_ID_FIELDNAME, id.getUuid().toString().toLowerCase() ),
+
+ FilterBuilders.rangeFilter( ENTITY_VERSION_FIELDNAME ).lt( version.timestamp() ) );
+
+ final DeleteByQueryResponse response =
+ esProvider.getClient().prepareDeleteByQuery( indexName ).setQuery( fqb ).execute().actionGet();
+
+ //error message needs to be retooled so that it describes the entity more throughly
+ logger.debug( "Deleted entity {}:{} with version {} from all index scopes with response status = {}",
+ new Object[] { id.getType(), id.getUuid(), version, response.status().toString() } );
+
+ checkDeleteByQueryResponse( fqb, response );
+ }
+
+
+ /**
+ * Validate the response doens't contain errors, if it does, fail fast at the first error we encounter
+ * @param query
+ * @param response
+ */
+ private void checkDeleteByQueryResponse( final QueryBuilder query, final DeleteByQueryResponse response ) {
+ for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) {
+ final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures();
+
+ for ( ShardOperationFailedException failedException : failures ) {
+ throw new IndexException( String.format(
+ "Unable to delete by query %s. Failed with code %d and reason %s on shard %s in index %s",
+ query.toString(), failedException.status(), failedException.reason(), failedException.shardId(),
+ failedException.index() ) );
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/43105f12/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index c7592bc..ee5557c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -50,7 +50,9 @@ public class IndexingUtils {
public static final String ENTITY_CONTEXT_FIELDNAME = "ug_context";
- public static final String ENTITYID_ID_FIELDNAME = "ug_entityId";
+ public static final String ENTITY_ID_FIELDNAME = "ug_entityId";
+
+ public static final String ENTITY_VERSION_FIELDNAME = "ug_entityVersion";
@@ -159,8 +161,8 @@ public class IndexingUtils {
//we need most specific mappings first since it's a stop on match algorithm
.startObject()
- .startObject( "context_template" )
- .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
+ .startObject( "entity_id_template" )
+ .field( "match", IndexingUtils.ENTITY_ID_FIELDNAME )
.field( "match_mapping_type", "string" )
.startObject( "mapping" ).field( "type", "string" )
.field( "index", "not_analyzed" )
@@ -170,7 +172,7 @@ public class IndexingUtils {
.startObject()
- .startObject( "context_template" )
+ .startObject( "entity_context_template" )
.field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
.field( "match_mapping_type", "string" )
.startObject( "mapping" ).field( "type", "string" )
@@ -178,6 +180,15 @@ public class IndexingUtils {
.endObject()
.endObject()
+ .startObject()
+ .startObject( "entity_version_template" )
+ .field( "match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" ).field( "type", "long" )
+ .endObject()
+ .endObject()
+ .endObject()
+
// any string with field name that starts with sa_ gets analyzed
.startObject()
.startObject( "template_1" ).field( "match", ANALYZED_STRING_PREFIX + "*" )
[19/50] incubator-usergrid git commit: Merge branch 'pr/92' into
two-dot-o-events. This closes #92
Posted by sf...@apache.org.
Merge branch 'pr/92' into two-dot-o-events. This closes #92
Conflicts:
stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5706b2c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5706b2c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5706b2c0
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 5706b2c0551bb64b1d1ad31718b16026a1d226ce
Parents: 5c4c24d e9f876c
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Nov 4 10:32:30 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Nov 4 10:32:30 2014 -0500
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../usergrid/corepersistence/GuiceModule.java | 9 ++
.../events/EntityVersionCreatedHandler.java | 17 ++-
.../corepersistence/StaleIndexCleanupTest.java | 33 ++++-
.../corepersistence/TestGuiceModule.java | 5 +
.../impl/EntityCollectionManagerImpl.java | 9 +-
.../impl/EntityVersionCreatedTask.java | 4 +-
.../impl/EntityVersionCreatedTaskTest.java | 132 +++++++++++++++----
.../index/impl/EsEntityIndexBatchImpl.java | 27 ++--
.../persistence/index/impl/IndexingUtils.java | 2 +
10 files changed, 189 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5706b2c0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5706b2c0/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 3dd63c0,1e64502..97a7288
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@@ -284,10 -310,12 +310,12 @@@ public class StaleIndexCleanupTest exte
* Test that the EntityDeleteImpl cleans up stale indexes on update. Ensures that when an
* entity is updated its old indexes are cleared from ElasticSearch.
*/
- @Test
+ @Test(timeout=10000)
public void testCleanupOnUpdate() throws Exception {
- logger.info("Started testCleanupOnUpdate()");
+ System.setProperty( "allow.stale.entities","false" );
+
+ logger.info( "Started testCleanupOnUpdate()" );
// TODO: turn off index cleanup on read
@@@ -337,16 -365,9 +365,17 @@@
// query Core Persistence directly for total number of result candidates
crs = queryCollectionCp("things", "select *");
- Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
+ Assert.assertEquals( "Expect candidates without earlier stale entities", numEntities, crs.size() );
+ // wait for indexes to be cleared for the deleted entities
+ do {
+ Thread.sleep(100);
+ crs = queryCollectionCp("things", "select *");
+ } while ( crs.size() > 0 );
+
+ // will never get here if test times out
+ Assert.assertEquals( "Expect no candidates", 0, crs.size() );
+ System.clearProperty( "allow.stale.entities" );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5706b2c0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
[21/50] incubator-usergrid git commit: Removed queue from NamedTask
Executor and made it do one event at a time. Added Rejected tests to Entity
Events.
Posted by sf...@apache.org.
Removed queue from NamedTask Executor and made it do one event at a time. Added Rejected tests to Entity Events.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a49b4449
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a49b4449
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a49b4449
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: a49b444961c0fc498b2bee0f396c8ca56b199a31
Parents: e9f876c
Author: grey <gr...@apigee.com>
Authored: Tue Nov 4 12:12:24 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Nov 4 12:12:24 2014 -0800
----------------------------------------------------------------------
.../impl/EntityVersionCleanupTaskTest.java | 129 ++++++-------------
.../impl/EntityVersionCreatedTaskTest.java | 46 +++++++
.../core/task/NamedTaskExecutorImpl.java | 114 +++++++++++++++-
.../core/task/NamedTaskExecutorImplTest.java | 32 +++++
4 files changed, 228 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a49b4449/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
index 621db20..0e86027 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
@@ -639,168 +639,123 @@ public class EntityVersionCleanupTaskTest {
latch.await();
}
-
/**
* Tests that our task will run in the caller if there's no threads, ensures that the task runs
*/
- @Ignore("Test is a work in progress")
@Test(timeout=10000)
- public void runsWhenRejected()
+ public void singleListenerSingleVersionRejected()
throws ExecutionException, InterruptedException, ConnectionException {
- /**
- * only 1 thread on purpose, we want to saturate the task
- */
- final TaskExecutor taskExecutor = new NamedTaskExecutorImpl( "test", 1, 0 );
+ final TaskExecutor taskExecutor = new NamedTaskExecutorImpl( "test", 0, 0 );
final SerializationFig serializationFig = mock( SerializationFig.class );
when( serializationFig.getBufferSize() ).thenReturn( 10 );
- final MvccEntitySerializationStrategy mvccEntitySerializationStrategy =
+ final MvccEntitySerializationStrategy ess =
mock( MvccEntitySerializationStrategy.class );
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy =
mock( MvccLogEntrySerializationStrategy.class );
- final Keyspace keyspace1 = mock( Keyspace.class );
- final Keyspace keyspace2 = mock( Keyspace.class );
+ final Keyspace keyspace = mock( Keyspace.class );
final MutationBatch entityBatch = mock( MutationBatch.class );
final MutationBatch logBatch = mock( MutationBatch.class );
- when( keyspace1.prepareMutationBatch() )
- .thenReturn( mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch )
- .thenReturn( logBatch );
+ when( keyspace.prepareMutationBatch() )
+ .thenReturn( mock( MutationBatch.class ) ) // don't care what happens to this one
+ .thenReturn( entityBatch )
+ .thenReturn( logBatch );
- when( keyspace2.prepareMutationBatch() )
- .thenReturn( mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch )
- .thenReturn( logBatch );
//create a latch for the event listener, and add it to the list of events
- final int sizeToReturn = 10;
-
-
- final int listenerCount = 2;
+ final int sizeToReturn = 1;
- final CountDownLatch latch = new CountDownLatch(
- sizeToReturn/serializationFig.getBufferSize() * listenerCount );
- final Semaphore waitSemaphore = new Semaphore( 0 );
+ final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+ final EntityVersionDeletedTest eventListener = new EntityVersionDeletedTest( latch );
- final SlowListener slowListener = new SlowListener( latch, waitSemaphore );
- final EntityVersionDeletedTest runListener = new EntityVersionDeletedTest( latch );
+ final Set<EntityVersionDeleted> listeners = new HashSet<EntityVersionDeleted>();
+ listeners.add( eventListener );
final Id applicationId = new SimpleId( "application" );
- final CollectionScope appScope = new CollectionScopeImpl(
+ final CollectionScope appScope = new CollectionScopeImpl(
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
//mock up a single log entry for our first test
- final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
+ final LogEntryMock logEntryMock = LogEntryMock.createLogEntryMock(
mvccLogEntrySerializationStrategy, appScope, entityId, sizeToReturn + 1 );
final UUID version = logEntryMock.getEntries().iterator().next().getVersion();
+
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy =
mock( UniqueValueSerializationStrategy.class );
-
- EntityVersionCleanupTask firstTask = new EntityVersionCleanupTask(
- serializationFig,
- mvccLogEntrySerializationStrategy,
- mvccEntitySerializationStrategy,
- uniqueValueSerializationStrategy,
- keyspace1,
- Sets.newSet( (EntityVersionDeleted)runListener ),
- appScope,
- entityId,
- version );
-
-
- //change the listeners to one that is just invoked quickly
-
-
- EntityVersionCleanupTask secondTask = new EntityVersionCleanupTask(
- serializationFig,
- mvccLogEntrySerializationStrategy,
- mvccEntitySerializationStrategy,
- uniqueValueSerializationStrategy,
- keyspace2,
- Sets.newSet( (EntityVersionDeleted)runListener ),
- appScope,
- entityId,
- version );
-
+ EntityVersionCleanupTask cleanupTask =
+ new EntityVersionCleanupTask( serializationFig,
+ mvccLogEntrySerializationStrategy,
+ ess,
+ uniqueValueSerializationStrategy,
+ keyspace,
+ listeners,
+ appScope,
+ entityId,
+ version
+ );
final MutationBatch batch = mock( MutationBatch.class );
//set up returning a mutator
- when( mvccEntitySerializationStrategy
- .delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
+ when(ess.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
.thenReturn( batch );
-
when( mvccLogEntrySerializationStrategy
.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
.thenReturn( batch );
- //start the task
- ListenableFuture<Void> future1 = taskExecutor.submit( firstTask );
+ final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- //now start another task while the slow running task is running
- ListenableFuture<Void> future2 = taskExecutor.submit( secondTask );
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- //get the second task, we shouldn't have been able to queue it,
- // therefore it should just run in process
- future2.get();
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- /**
- * While we're not done, release latches every 200 ms
- */
- while ( !future1.isDone() ) {
- Thread.sleep( 200 );
- waitSemaphore.release( listenerCount );
- }
+ when( ess.load( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
+ .thenReturn(mel.iterator() );
- //wait for the task
- future1.get();
- //we deleted the version
- //verify we deleted everything
+ //start the task
+ ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
+ //wait for the task
+ future.get();
//we deleted the version
- //verify we deleted everything
- verify( logBatch, times( sizeToReturn* 2 ) ).mergeShallow( any( MutationBatch.class ) );
-
- verify( entityBatch, times( sizeToReturn * 2) ).mergeShallow( any( MutationBatch.class ) );
-
-
- verify( logBatch, times(2) ).execute();
-
- verify( entityBatch, times(2) ).execute();
+ //verify it was run
+ verify( entityBatch ).execute();
+ verify( logBatch ).execute();
//the latch was executed
latch.await();
}
-
private static class EntityVersionDeletedTest implements EntityVersionDeleted {
final CountDownLatch invocationLatch;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a49b4449/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 9d72665..24ea280 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -200,6 +200,52 @@ public class EntityVersionCreatedTaskTest {
verify( listeners ).iterator();
}
+ @Test(timeout=10000)
+ public void oneListenerRejected()
+ throws ExecutionException, InterruptedException, ConnectionException {
+
+ // create a latch for the event listener, and add it to the list of events
+
+ final TaskExecutor taskExecutor = new NamedTaskExecutorImpl( "test", 0, 0 );
+
+ final int sizeToReturn = 1;
+
+ final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+
+ final EntityVersionCreatedTest eventListener = new EntityVersionCreatedTest(latch);
+
+ final Set<EntityVersionCreated> listeners = mock( Set.class );
+ final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
+
+ when ( listeners.size()).thenReturn( 1 );
+ when ( listeners.iterator()).thenReturn( helper );
+ when ( helper.next() ).thenReturn( eventListener );
+
+ final Id applicationId = new SimpleId( "application" );
+
+ final CollectionScope appScope = new CollectionScopeImpl(
+ applicationId, applicationId, "users" );
+
+ final Id entityId = new SimpleId( "user" );
+ final Entity entity = new Entity( entityId );
+
+ // start the task
+
+ EntityVersionCreatedTask entityVersionCreatedTask =
+ new EntityVersionCreatedTask( appScope, listeners, entity);
+
+ ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
+
+ // wait for the task
+ future.get();
+
+ //mocked listener makes sure that the task is called
+ verify( listeners ).size();
+ verify( listeners ).iterator();
+ verify( helper ).next();
+
+ }
+
private static class EntityVersionCreatedTest implements EntityVersionCreated {
final CountDownLatch invocationLatch;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a49b4449/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
index b18687a..a022c08 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImpl.java
@@ -1,16 +1,24 @@
package org.apache.usergrid.persistence.core.task;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.ForkJoinWorkerThread;
+import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.Nullable;
@@ -36,7 +44,6 @@ public class NamedTaskExecutorImpl implements TaskExecutor {
private final String name;
private final int poolSize;
- private final int queueLength;
/**
@@ -47,17 +54,24 @@ public class NamedTaskExecutorImpl implements TaskExecutor {
public NamedTaskExecutorImpl( final String name, final int poolSize, final int queueLength ) {
Preconditions.checkNotNull( name );
Preconditions.checkArgument( name.length() > 0, "name must have a length" );
- Preconditions.checkArgument( poolSize > 0, "poolSize must be > than 0" );
+ Preconditions.checkArgument( poolSize > -1, "poolSize must be > than -1" );
Preconditions.checkArgument( queueLength > -1, "queueLength must be 0 or more" );
this.name = name;
this.poolSize = poolSize;
- this.queueLength = queueLength;
- final BlockingQueue<Runnable> queue =
- queueLength > 0 ? new ArrayBlockingQueue<Runnable>( queueLength ) : new SynchronousQueue<Runnable>();
+ //The user has chosen to disable asynchronous execution, to create an executor service that will reject all requests
+ if(poolSize == 0){
+ executorService = MoreExecutors.listeningDecorator( new RejectingExecutorService());
+ }
+
+ //queue executions as normal
+ else {
+ final BlockingQueue<Runnable> queue =
+ queueLength > 0 ? new ArrayBlockingQueue<Runnable>( queueLength ) : new SynchronousQueue<Runnable>();
- executorService = MoreExecutors.listeningDecorator( new MaxSizeThreadPool( queue ) );
+ executorService = MoreExecutors.listeningDecorator( new MaxSizeThreadPool( queue ) );
+ }
}
@@ -164,4 +178,92 @@ public class NamedTaskExecutorImpl implements TaskExecutor {
}
}
+
+
+ /**
+ * Executor implementation that simply rejects all incoming tasks
+ */
+ private static final class RejectingExecutorService implements ExecutorService{
+
+ @Override
+ public void shutdown() {
+
+ }
+
+
+ @Override
+ public List<Runnable> shutdownNow() {
+ return Collections.EMPTY_LIST;
+ }
+
+
+ @Override
+ public boolean isShutdown() {
+ return false;
+ }
+
+
+ @Override
+ public boolean isTerminated() {
+ return false;
+ }
+
+
+ @Override
+ public boolean awaitTermination( final long timeout, final TimeUnit unit ) throws InterruptedException {
+ return false;
+ }
+
+
+ @Override
+ public <T> Future<T> submit( final Callable<T> task ) {
+ throw new RejectedExecutionException("No Asynchronous tasks allowed");
+ }
+
+
+ @Override
+ public <T> Future<T> submit( final Runnable task, final T result ) {
+ throw new RejectedExecutionException("No Asynchronous tasks allowed");
+ }
+
+
+ @Override
+ public Future<?> submit( final Runnable task ) {
+ throw new RejectedExecutionException("No Asynchronous tasks allowed");
+ }
+
+
+ @Override
+ public <T> List<Future<T>> invokeAll( final Collection<? extends Callable<T>> tasks )
+ throws InterruptedException {
+ throw new RejectedExecutionException("No Asynchronous tasks allowed");
+ }
+
+
+ @Override
+ public <T> List<Future<T>> invokeAll( final Collection<? extends Callable<T>> tasks, final long timeout,
+ final TimeUnit unit ) throws InterruptedException {
+ throw new RejectedExecutionException("No Asynchronous tasks allowed");
+ }
+
+
+ @Override
+ public <T> T invokeAny( final Collection<? extends Callable<T>> tasks )
+ throws InterruptedException, ExecutionException {
+ throw new RejectedExecutionException("No Asynchronous tasks allowed");
+ }
+
+
+ @Override
+ public <T> T invokeAny( final Collection<? extends Callable<T>> tasks, final long timeout, final TimeUnit unit )
+ throws InterruptedException, ExecutionException, TimeoutException {
+ throw new RejectedExecutionException("No Asynchronous tasks allowed");
+ }
+
+
+ @Override
+ public void execute( final Runnable command ) {
+ throw new RejectedExecutionException("No Asynchronous tasks allowed");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a49b4449/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
index 34f57e5..65189f1 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/task/NamedTaskExecutorImplTest.java
@@ -185,6 +185,38 @@ public class NamedTaskExecutorImplTest {
}
+ @Test
+ public void rejectingTaskExecutor() throws InterruptedException {
+
+ final int threadPoolSize = 0;
+ final int queueSize = 0;
+
+ final TaskExecutor executor = new NamedTaskExecutorImpl( "jobSuccess", threadPoolSize, queueSize );
+
+ final CountDownLatch exceptionLatch = new CountDownLatch( 0 );
+ final CountDownLatch rejectedLatch = new CountDownLatch( 1 );
+ final CountDownLatch runLatch = new CountDownLatch( 0 );
+
+
+ //now submit the second task
+
+
+
+ final TestTask<Void> testTask = new TestTask<Void>( exceptionLatch, rejectedLatch, runLatch ) {};
+
+ executor.submit( testTask );
+
+
+ //should be immediately rejected
+ rejectedLatch.await( 1000, TimeUnit.MILLISECONDS );
+
+ //if we get here we've been rejected, just double check we didn't run
+
+ assertEquals( 0l, exceptionLatch.getCount() );
+ assertEquals( 0l, runLatch.getCount() );
+ }
+
+
private static abstract class TestTask<V> implements Task<V> {
private final List<Throwable> exceptions;
[22/50] incubator-usergrid git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-events
# By Dave Johnson
# Via Dave Johnson
* 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid:
Add new ignore_error argument to /status end-point, which defaults to true and, also added Index health check to the /org/app/status end-point.
There is now a new option on the /status end-point. If you specify "ignore_errors=false" the the /status end-point will return an HTTP 500 error if there is a problem connecting to Cassandra or of the query index is status RED for the management app.
Oops
Set shard and replica size at Index creation time and not client creation time.
Conflicts:
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e759a798
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e759a798
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e759a798
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: e759a7980c0b75cca2b4bcc31fa6d920da97db81
Parents: a49b444 5285f3d
Author: grey <gr...@apigee.com>
Authored: Tue Nov 4 15:33:38 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Nov 4 15:33:38 2014 -0800
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 12 +-
.../corepersistence/CpEntityManagerFactory.java | 30 ++--
.../HybridEntityManagerFactory.java | 10 +-
.../usergrid/persistence/EntityManager.java | 6 +
.../persistence/EntityManagerFactory.java | 25 ++--
.../cassandra/EntityManagerFactoryImpl.java | 10 +-
.../cassandra/EntityManagerImpl.java | 6 +
.../collection/EntityCollectionManager.java | 6 +-
.../impl/EntityCollectionManagerImpl.java | 7 +-
.../collection/EntityCollectionManagerIT.java | 3 +-
.../usergrid/persistence/core/util/Health.java | 23 ++++
.../usergrid/persistence/index/EntityIndex.java | 13 +-
.../index/impl/EsEntityIndexImpl.java | 55 ++++++--
.../persistence/index/impl/EsProvider.java | 14 +-
.../impl/EntityConnectionIndexImplTest.java | 14 --
.../persistence/index/impl/EntityIndexTest.java | 36 +++--
.../rest/JacksonCustomMapperProvider.java | 7 +-
.../org/apache/usergrid/rest/RootResource.java | 42 +++++-
.../rest/filters/HealthCheckFilter.java | 68 ----------
.../applications/ApplicationResource.java | 136 +++++++++++++------
stack/rest/src/main/webapp/WEB-INF/web.xml | 10 --
21 files changed, 309 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e759a798/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e759a798/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e759a798/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e759a798/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 16046b3,bb75cc9..437c0d3
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -60,9 -62,8 +60,10 @@@ import com.netflix.astyanax.connectionp
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
+ import org.apache.usergrid.persistence.core.util.Health;
import rx.Observable;
import rx.Subscriber;
[14/50] incubator-usergrid git commit: test fixes
Posted by sf...@apache.org.
test fixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a3cfd56b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a3cfd56b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a3cfd56b
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: a3cfd56b6d4a0e5043549a6c64b7067554a2167b
Parents: 4a36aa1
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Oct 31 15:55:19 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Oct 31 15:55:19 2014 -0400
----------------------------------------------------------------------
.../usergrid/corepersistence/StaleIndexCleanupTest.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a3cfd56b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index ff740af..3dd63c0 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -208,7 +208,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
* Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an
* entity is deleted its old indexes are cleared from ElasticSearch.
*/
- @Test
+ @Test(timeout=10000)
public void testCleanupOnDelete() throws Exception {
logger.info("Started testStaleIndexCleanup()");
@@ -274,7 +274,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
do {
Thread.sleep(100);
crs = queryCollectionCp("things", "select *");
- } while ( crs.size() > 0 || count++ < 14 );
+ } while ( crs.size() > 0 && count++ < 14 );
Assert.assertEquals( "Expect no candidates", 0, crs.size() );
}
@@ -284,7 +284,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
* Test that the EntityDeleteImpl cleans up stale indexes on update. Ensures that when an
* entity is updated its old indexes are cleared from ElasticSearch.
*/
- @Test
+ @Test(timeout=10000)
public void testCleanupOnUpdate() throws Exception {
logger.info("Started testCleanupOnUpdate()");
@@ -340,12 +340,12 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
// wait for indexes to be cleared for the deleted entities
- count = 0;
do {
Thread.sleep(100);
crs = queryCollectionCp("things", "select *");
- } while ( crs.size() > 0 || count++ < 14 );
+ } while ( crs.size() > 0 );
+ // will never get here if test times out
Assert.assertEquals( "Expect no candidates", 0, crs.size() );
}
[25/50] incubator-usergrid git commit: Fixed wrong method call in
test.
Posted by sf...@apache.org.
Fixed wrong method call in test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e826cc20
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e826cc20
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e826cc20
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: e826cc201cddc0f6f1d919e12f6c4cbb2e92aad2
Parents: 216e050
Author: grey <gr...@apigee.com>
Authored: Tue Nov 4 15:55:52 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Nov 4 15:55:52 2014 -0800
----------------------------------------------------------------------
.../persistence/collection/impl/EntityVersionCleanupTaskTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e826cc20/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
index 1c7d510..eae1c54 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
@@ -742,7 +742,7 @@ public class EntityVersionCleanupTaskTest {
try {
- cleanupTask.call();
+ cleanupTask.rejected();
}catch(Exception e){
Assert.fail(e.getMessage());
}
[34/50] incubator-usergrid git commit: Fix merge issue.
Posted by sf...@apache.org.
Fix merge issue.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/32276938
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/32276938
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/32276938
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 32276938d6a4ea986a15fa7bc7c2970a2a83ce41
Parents: c69f0ec
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Nov 24 09:40:22 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Nov 24 09:40:22 2014 -0500
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 17 ++-
.../persistence/index/impl/IndexingUtils.java | 115 ++++++++++---------
2 files changed, 67 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32276938/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 53aaf54..aa797e2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -51,7 +51,6 @@ import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
-import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -72,7 +71,6 @@ import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -82,6 +80,7 @@ import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
@@ -462,8 +461,7 @@ public class EsEntityIndexImpl implements EntityIndex {
public void deleteAllVersionsOfEntity( Id entityId ) {
final TermQueryBuilder tqb =
- QueryBuilders.termQuery( ENTITY_ID_FIELDNAME, entityId.getUuid().toString().toLowerCase() );
-
+ QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, entityId.getUuid().toString().toLowerCase() );
final DeleteByQueryResponse response =
esProvider.getClient().prepareDeleteByQuery( indexName ).setQuery( tqb ).execute().actionGet();
@@ -481,9 +479,8 @@ public class EsEntityIndexImpl implements EntityIndex {
public void deletePreviousVersions( final Id id, final UUID version ) {
final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
- QueryBuilders.termQuery( ENTITY_ID_FIELDNAME, id.getUuid().toString().toLowerCase() ),
-
- FilterBuilders.rangeFilter( ENTITY_VERSION_FIELDNAME ).lt( version.timestamp() ) );
+ QueryBuilders.termQuery( IndexingUtils.ENTITYID_ID_FIELDNAME, id.getUuid().toString().toLowerCase() ),
+ FilterBuilders.rangeFilter( IndexingUtils.ENTITY_VERSION_FIELDNAME ).lt( version.timestamp() ) );
final DeleteByQueryResponse response =
esProvider.getClient().prepareDeleteByQuery( indexName ).setQuery( fqb ).execute().actionGet();
@@ -507,9 +504,9 @@ public class EsEntityIndexImpl implements EntityIndex {
for ( ShardOperationFailedException failedException : failures ) {
throw new IndexException( String.format(
- "Unable to delete by query %s. Failed with code %d and reason %s on shard %s in index %s",
- query.toString(), failedException.status(), failedException.reason(), failedException.shardId(),
- failedException.index() ) );
+ "Unable to delete by query %s. Failed with code %d and reason %s on shard %s in index %s",
+ query.toString(), failedException.status(), failedException.reason(), failedException.shardId(),
+ failedException.index() ) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32276938/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 30b05e4..b26ca62 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -41,14 +41,17 @@ public class IndexingUtils {
// These are not allowed in document type names: _ . , | #
public static final String SEPARATOR = "__";
- /**
- * Reserved UG fields.
- */
+
+ //
+ // Reserved UG fields.
+ //
public static final String ENTITY_CONTEXT_FIELDNAME = "ug_context";
public static final String ENTITYID_ID_FIELDNAME = "ug_entityId";
+ public static final String ENTITY_VERSION_FIELDNAME = "ug_entityVersion";
+
/**
* Create our sub scope. This is the ownerUUID + type
@@ -155,73 +158,75 @@ public class IndexingUtils {
// we need most specific mappings first since it's a stop on match algorithm
.startObject()
- .startObject( "entity_id_template" )
- .field( "match", IndexingUtils.ENTITY_ID_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "string" )
- .field( "index", "not_analyzed" )
- .endObject()
- .endObject()
- .endObject()
-
- .startObject()
- .startObject( "entity_context_template" )
- .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" )
- .field( "type", "string" )
- .field( "index", "not_analyzed" ).endObject()
- .endObject()
- .endObject()
- .startObject()
- .startObject( "entity_version_template" )
- .field( "match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
+ .startObject( "entity_id_template" )
+ .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
.field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "long" )
- .endObject()
+ .startObject( "mapping" ).field( "type", "string" )
+ .field( "index", "not_analyzed" )
+ .endObject()
.endObject()
.endObject()
- // any string with field name that starts with sa_ gets analyzed
- .startObject()
- .startObject( "template_1" )
- .field( "match", ANALYZED_STRING_PREFIX + "*" )
- .field( "match_mapping_type", "string" ).startObject( "mapping" )
- .field( "type", "string" )
- .field( "index", "analyzed" )
+ .startObject()
+ .startObject( "entity_context_template" )
+ .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "not_analyzed" ).endObject()
+ .endObject()
.endObject()
- .endObject()
- .endObject()
-
- // all other strings are not analyzed
- .startObject()
- .startObject( "template_2" )
- //todo, should be string prefix, remove 2 field mapping
- .field( "match", "*" )
- .field( "match_mapping_type", "string" )
- .startObject( "mapping" )
- .field( "type", "string" )
- .field( "index", "not_analyzed" )
+
+ .startObject()
+ .startObject( "entity_version_template" )
+ .field( "match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" ).field( "type", "long" )
+ .endObject()
+ .endObject()
+ .endObject()
+
+ // any string with field name that starts with sa_ gets analyzed
+ .startObject()
+ .startObject( "template_1" )
+ .field( "match", ANALYZED_STRING_PREFIX + "*" )
+ .field( "match_mapping_type", "string" ).startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "analyzed" )
+ .endObject()
.endObject()
+
.endObject()
- .endObject()
- // fields names starting with go_ get geo-indexed
- .startObject()
- .startObject( "template_3" )
- .field( "match", GEO_PREFIX + "location" )
+ // all other strings are not analyzed
+ .startObject()
+ .startObject( "template_2" )
+ //todo, should be string prefix, remove 2 field mapping
+ .field( "match", "*" )
+ .field( "match_mapping_type", "string" )
.startObject( "mapping" )
- .field( "type", "geo_point" )
+ .field( "type", "string" )
+ .field( "index", "not_analyzed" )
.endObject()
+ .endObject()
+ .endObject()
+
+ // fields names starting with go_ get geo-indexed
+ .startObject()
+ .startObject( "template_3" )
+ .field( "match", GEO_PREFIX + "location" )
+ .startObject( "mapping" )
+ .field( "type", "geo_point" )
+ .endObject()
+ .endObject()
.endObject()
- .endObject()
- .endArray()
+ .endArray()
- .endObject()
+ .endObject()
- .endObject();
+ .endObject();
return builder;
}
[27/50] incubator-usergrid git commit: Fix merge problem.
Posted by sf...@apache.org.
Fix merge problem.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8457acf2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8457acf2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8457acf2
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 8457acf2876bdde431f8042e7928a8be94e3a38c
Parents: bce8142
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Nov 7 10:16:53 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Nov 7 10:16:53 2014 -0500
----------------------------------------------------------------------
.../persistence/index/impl/EsEntityIndexBatchImpl.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8457acf2/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index a0344e0..b1e3b33 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -258,9 +258,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
try {
responses = request.execute().actionGet();
- }catch(Throwable t){
- log.error( "Unable to communicate with elasticsearch" );
- failureMonitor.fail( "Unable to execute batch", t );
+ } catch (Throwable t) {
+ logger.error("Unable to communicate with elasticsearch");
+ failureMonitor.fail("Unable to execute batch", t);
throw t;
}
[50/50] incubator-usergrid git commit: merge
Posted by sf...@apache.org.
merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/81edb677
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/81edb677
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/81edb677
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 81edb677fde8d096fea2262a05a3b0136ce65923
Parents: 615ac49 37940e3
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Dec 22 08:22:43 2014 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Dec 22 08:22:43 2014 -0700
----------------------------------------------------------------------
stack/core/.test-tdoe-5thread.log.swp | Bin 0 -> 28672 bytes
.../corepersistence/CpEntityDeleteListener.java | 101 -----
.../CpEntityIndexDeleteListener.java | 98 ----
.../corepersistence/CpEntityManager.java | 124 ++---
.../corepersistence/CpRelationManager.java | 35 +-
.../usergrid/corepersistence/CpSetup.java | 16 +-
.../usergrid/corepersistence/GuiceModule.java | 43 +-
.../HybridEntityManagerFactory.java | 4 +
.../events/EntityDeletedHandler.java | 68 +++
.../events/EntityVersionCreatedHandler.java | 69 +++
.../events/EntityVersionDeletedHandler.java | 102 +++++
.../results/FilteringLoader.java | 7 +-
.../CpEntityDeleteListenerTest.java | 91 ----
.../CpEntityIndexDeleteListenerTest.java | 111 -----
.../corepersistence/StaleIndexCleanupTest.java | 219 ++++++++-
.../corepersistence/TestGuiceModule.java | 9 +-
.../migration/GraphShardVersionMigrationIT.java | 5 +-
.../usergrid/persistence/CollectionIT.java | 15 +-
.../query/AbstractIteratingQueryIT.java | 2 +-
.../resources/usergrid-custom-test.properties | 3 +
.../collection/EntityCollectionManager.java | 13 +-
.../collection/EntityDeletedFactory.java | 34 ++
.../persistence/collection/EntitySet.java | 4 +-
.../collection/EntityVersionCleanupFactory.java | 35 ++
.../collection/EntityVersionCreatedFactory.java | 31 ++
.../persistence/collection/VersionSet.java | 22 +-
.../collection/event/EntityDeleted.java | 5 +-
.../collection/event/EntityVersionCreated.java | 14 +-
.../collection/event/EntityVersionDeleted.java | 11 +-
.../collection/guice/CollectionModule.java | 24 +-
.../collection/impl/CollectionScopeImpl.java | 4 +-
.../impl/EntityCollectionManagerImpl.java | 86 +++-
.../collection/impl/EntityDeletedTask.java | 148 ++++++
.../impl/EntityVersionCleanupTask.java | 224 +++++----
.../impl/EntityVersionCreatedTask.java | 122 +++++
.../mvcc/stage/delete/MarkCommit.java | 137 +++---
.../impl/EntityVersionSerializer.java | 3 -
.../collection/guice/TestCollectionModule.java | 4 -
.../impl/EntityVersionCleanupTaskTest.java | 450 ++++++++++++-------
.../impl/EntityVersionCreatedTaskTest.java | 244 ++++++++++
.../core/task/NamedTaskExecutorImpl.java | 141 +++++-
.../core/task/NamedTaskExecutorImplTest.java | 56 ++-
.../usergrid/persistence/index/EntityIndex.java | 20 +-
.../persistence/index/EntityIndexBatch.java | 7 +-
.../index/impl/EsEntityIndexBatchImpl.java | 9 +-
.../index/impl/EsEntityIndexImpl.java | 142 ++++--
.../persistence/index/impl/IndexingUtils.java | 105 +++--
.../apache/usergrid/rest/TestContextSetup.java | 6 +-
.../collection/DuplicateNameIT.java | 2 +-
.../rest/test/resource/TestContext.java | 3 +-
50 files changed, 2161 insertions(+), 1067 deletions(-)
----------------------------------------------------------------------
[08/50] incubator-usergrid git commit: Merge branch 'pr/91' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'pr/91' into two-dot-o-events
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f8131365
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f8131365
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f8131365
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: f8131365e23fc6896dd0d6b5c00c4183e4878beb
Parents: bd2b127 389bee7
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 30 09:38:25 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 30 09:38:25 2014 -0400
----------------------------------------------------------------------
.../events/EntityVersionCreatedImpl.java | 49 ++++
.../collection/EntityVersionCreatedFactory.java | 31 +++
.../collection/guice/CollectionModule.java | 16 +-
.../impl/EntityCollectionManagerImpl.java | 20 +-
.../impl/EntityVersionCreatedTask.java | 150 +++++++++++
.../impl/EntityVersionCreatedTaskTest.java | 270 +++++++++++++++++++
.../persistence/index/EntityIndexBatch.java | 12 +-
.../index/impl/EsEntityIndexBatchImpl.java | 22 ++
8 files changed, 546 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
[18/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Conflicts:
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5c4c24d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5c4c24d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5c4c24d1
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 5c4c24d1a9c558ec59b7171e23e6f4c8ac01a6a5
Parents: a3cfd56 5285f3d
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Nov 4 10:19:03 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Nov 4 10:19:03 2014 -0500
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 12 +-
.../corepersistence/CpEntityManagerFactory.java | 30 ++--
.../HybridEntityManagerFactory.java | 10 +-
.../usergrid/persistence/EntityManager.java | 6 +
.../persistence/EntityManagerFactory.java | 25 ++--
.../cassandra/EntityManagerFactoryImpl.java | 10 +-
.../cassandra/EntityManagerImpl.java | 6 +
.../collection/EntityCollectionManager.java | 5 +-
.../impl/EntityCollectionManagerImpl.java | 7 +-
.../collection/EntityCollectionManagerIT.java | 3 +-
.../usergrid/persistence/core/util/Health.java | 23 ++++
.../usergrid/persistence/index/EntityIndex.java | 13 +-
.../index/impl/EsEntityIndexImpl.java | 55 ++++++--
.../persistence/index/impl/EsProvider.java | 14 +-
.../impl/EntityConnectionIndexImplTest.java | 14 --
.../persistence/index/impl/EntityIndexTest.java | 36 +++--
.../rest/JacksonCustomMapperProvider.java | 7 +-
.../org/apache/usergrid/rest/RootResource.java | 42 +++++-
.../rest/filters/HealthCheckFilter.java | 68 ----------
.../applications/ApplicationResource.java | 136 +++++++++++++------
stack/rest/src/main/webapp/WEB-INF/web.xml | 10 --
21 files changed, 309 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c4c24d1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c4c24d1/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c4c24d1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 0205824,534d7a6..266b0f9
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@@ -20,7 -20,7 +20,8 @@@ package org.apache.usergrid.persistence
import java.util.Collection;
+ import org.apache.usergrid.persistence.core.util.Health;
+import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5c4c24d1/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 26ab6aa,bb75cc9..ea6c06f
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -60,9 -62,8 +60,10 @@@ import com.netflix.astyanax.connectionp
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+ import org.apache.usergrid.persistence.core.util.Health;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
import rx.Observable;
import rx.Subscriber;
[12/50] incubator-usergrid git commit: Use correct index name
Posted by sf...@apache.org.
Use correct index name
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7000bd9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7000bd9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7000bd9c
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 7000bd9c424a0ada98b03d0564385ae0b6933daf
Parents: 7dec37b
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Oct 31 14:00:44 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Oct 31 14:00:44 2014 -0400
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7000bd9c/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index ca7f666..aedee2b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -203,7 +203,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
TermQueryBuilder tqb = QueryBuilders.termQuery(
STRING_PREFIX + ENTITYID_FIELDNAME, entityId.getUuid().toString().toLowerCase());
- DeleteByQueryResponse response = client.prepareDeleteByQuery("test")
+ DeleteByQueryResponse response = client.prepareDeleteByQuery( indexName )
.setQuery( tqb ).execute().actionGet();
logger.debug("Deleted entity {}:{} from all index scopes with response status = {}",
@@ -222,7 +222,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
( STRING_PREFIX + ENTITYID_FIELDNAME,entity.getId().getUuid().toString().toLowerCase() ),
FilterBuilders.rangeFilter("version").lt( entity.getId().getUuid().timestamp() ));
- DeleteByQueryResponse response = client.prepareDeleteByQuery("test")
+ DeleteByQueryResponse response = client.prepareDeleteByQuery( indexName )
.setQuery( fqb ).execute().actionGet();
logger.debug("Deleted entity {}:{} from all index scopes with response status = {}",
[07/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bd2b1271
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bd2b1271
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bd2b1271
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: bd2b127188b1140b3d9a4d3e30c4df90ad92285b
Parents: 9153eb9 97ee70f
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 30 09:38:15 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 30 09:38:15 2014 -0400
----------------------------------------------------------------------
.../notifications/apns/APNsAdapter.java | 37 ++++++++++----------
.../notifications/apns/EntityPushManager.java | 5 +--
2 files changed, 22 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
[06/50] incubator-usergrid git commit: Re-added in the timeout and
removed the previous code that added complexity.
Posted by sf...@apache.org.
Re-added in the timeout and removed the previous code that added complexity.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/389bee72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/389bee72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/389bee72
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 389bee7282f3c60c77e8998d1ec9ea5807f316d2
Parents: da38f0b
Author: grey <gr...@apigee.com>
Authored: Wed Oct 29 16:46:29 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Wed Oct 29 16:46:29 2014 -0700
----------------------------------------------------------------------
.../persistence/collection/impl/EntityVersionCreatedTask.java | 5 +----
.../collection/impl/EntityVersionCreatedTaskTest.java | 4 ++--
2 files changed, 3 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/389bee72/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
index fdfb551..256e65f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
@@ -122,10 +122,7 @@ public class EntityVersionCreatedTask implements Task<Void> {
}
if ( listenerSize == 1 ) {
- EntityVersionCreated entityVersionCreated = listeners.iterator().next();
-
- entityVersionCreated.versionCreated( collectionScope,entity );
-
+ listeners.iterator().next().versionCreated( collectionScope, entity );
return;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/389bee72/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 9f8d77f..21895ba 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -134,8 +134,8 @@ public class EntityVersionCreatedTaskTest {
}
- @Test//(timeout=10000)
- public void oneListener()//why does it matter if it has a version or not without a listener
+ @Test(timeout=10000)
+ public void oneListener()
throws ExecutionException, InterruptedException, ConnectionException {
[20/50] incubator-usergrid git commit: Fixes to
StaleEntityCleapupTest, which is looking good now, also some import cleanups.
Posted by sf...@apache.org.
Fixes to StaleEntityCleapupTest, which is looking good now, also some import cleanups.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3d3bbedc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3d3bbedc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3d3bbedc
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 3d3bbedc2da0f84c9e1180fed17c9fd39fc1a605
Parents: 5706b2c
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Nov 4 11:11:33 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Nov 4 11:11:33 2014 -0500
----------------------------------------------------------------------
.../usergrid/corepersistence/GuiceModule.java | 1 -
.../events/EntityVersionCreatedHandler.java | 9 +----
.../corepersistence/StaleIndexCleanupTest.java | 39 ++++++--------------
.../corepersistence/TestGuiceModule.java | 2 -
.../impl/EntityVersionCreatedTaskTest.java | 20 ----------
.../index/impl/EsEntityIndexBatchImpl.java | 5 ---
6 files changed, 12 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3d3bbedc/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index 3b868b3..e63f272 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@ -17,7 +17,6 @@ package org.apache.usergrid.corepersistence;
import com.google.inject.AbstractModule;
import com.google.inject.multibindings.Multibinder;
-import com.google.inject.name.Names;
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3d3bbedc/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index ff09119..94a673a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -17,13 +17,8 @@
*/
package org.apache.usergrid.corepersistence.events;
-import java.util.Properties;
-
-import org.elasticsearch.common.inject.Guice;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.context.ConfigurableApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
import org.apache.usergrid.corepersistence.CpSetup;
@@ -34,8 +29,6 @@ import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.model.entity.Entity;
-import com.google.inject.Inject;
-import com.google.inject.name.Named;
/**
@@ -65,7 +58,7 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
EntityIndexBatch batch = ei.createBatch();
- if(System.getProperty( "allow.stale.entities","false" ).equals( "false" )) {
+ if ( System.getProperty( "allow.stale.entities", "false" ).equals( "false" )) {
batch.deindexPreviousVersions( entity );
batch.execute();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3d3bbedc/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 97a7288..dff2f59 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -18,20 +18,16 @@
package org.apache.usergrid.corepersistence;
import com.fasterxml.uuid.UUIDComparator;
-import com.google.inject.Guice;
-import com.google.inject.Inject;
-import com.google.inject.name.Named;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import java.util.Properties;
import java.util.UUID;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.usergrid.AbstractCoreIT;
-import org.apache.usergrid.CoreITSuite;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.count.SimpleBatcher;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
@@ -60,8 +56,6 @@ import org.junit.After;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.context.ConfigurableApplicationContext;
-import org.springframework.context.support.ClassPathXmlApplicationContext;
//need to create system properties in test that can get applied
@@ -75,11 +69,13 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
private static final long writeDelayMs = 50;
private static final long readDelayMs = 50;
+ Lock sequential = new ReentrantLock();
+
@Before
public void before() {
- //set the batcher to block the submit so we wait for results when testing
- System.setProperty( "allow.stale.entities","true" );
+ // if tests run in parallel there will likely be a conflict over the allow.stale.entities
+ sequential.lock();
}
@After
@@ -117,8 +113,6 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
assertTrue( "New version is greater than old",
UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
-
- assertEquals( 2, queryCollectionCp("things", "select *").size() );
}
@@ -131,7 +125,8 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
logger.info("Started testCleanupOnRead()");
- // TODO: turn off post processing stuff that cleans up stale entities
+ // turn off post processing stuff that cleans up stale entities
+ System.setProperty( "allow.stale.entities", "true" );
final EntityManager em = app.getEntityManager();
@@ -177,6 +172,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
maxVersions.add( toUpdate );
}
+
em.refreshIndex();
// query Core Persistence directly for total number of result candidates
@@ -239,7 +235,8 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
logger.info("Started testStaleIndexCleanup()");
- // TODO: turn off index cleanup on read
+ // turn off post processing stuff that cleans up stale entities
+ System.setProperty( "allow.stale.entities", "true" );
final EntityManager em = app.getEntityManager();
@@ -313,12 +310,8 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
@Test(timeout=10000)
public void testCleanupOnUpdate() throws Exception {
- System.setProperty( "allow.stale.entities","false" );
-
logger.info( "Started testCleanupOnUpdate()" );
- // TODO: turn off index cleanup on read
-
final EntityManager em = app.getEntityManager();
final int numEntities = 10;
@@ -366,16 +359,6 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
// query Core Persistence directly for total number of result candidates
crs = queryCollectionCp("things", "select *");
Assert.assertEquals( "Expect candidates without earlier stale entities", numEntities, crs.size() );
-
- // wait for indexes to be cleared for the deleted entities
- do {
- Thread.sleep(100);
- crs = queryCollectionCp("things", "select *");
- } while ( crs.size() > 0 );
-
- // will never get here if test times out
- Assert.assertEquals( "Expect no candidates", 0, crs.size() );
- System.clearProperty( "allow.stale.entities" );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3d3bbedc/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
index cdf4c01..7d1bd69 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
@@ -19,8 +19,6 @@
package org.apache.usergrid.corepersistence;
import com.google.inject.AbstractModule;
-import com.google.inject.Provides;
-import com.google.inject.name.Names;
public class TestGuiceModule extends AbstractModule {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3d3bbedc/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 9d72665..92bb3ce 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -17,45 +17,25 @@
*/
package org.apache.usergrid.persistence.collection.impl;
-import com.google.common.base.Optional;
import com.google.common.util.concurrent.ListenableFuture;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import java.util.ArrayList;
-import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
-import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.util.LogEntryMock;
import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.junit.AfterClass;
import org.junit.Test;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.same;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3d3bbedc/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 7f6cf70..ad15fff 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -23,20 +23,15 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
import java.util.Set;
import java.util.UUID;
-import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.FilteredQueryBuilder;
-import org.elasticsearch.index.query.QueryStringQueryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
[48/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a8cc5870
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a8cc5870
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a8cc5870
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: a8cc5870da6326e78b54ba358e06ed367953846c
Parents: a290e0d 31bb3f1
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Dec 11 13:59:36 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Dec 11 13:59:36 2014 -0500
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 18 +-
stack/core/pom.xml | 15 +-
.../corepersistence/CpEntityManagerFactory.java | 4 +-
.../org/apache/usergrid/AbstractCoreIT.java | 12 +-
.../apache/usergrid/ConcurrentCoreITSuite.java | 51 ----
.../usergrid/ConcurrentCoreIteratorITSuite.java | 83 ------
.../usergrid/ConcurrentCoreTestSuite.java | 46 ----
.../java/org/apache/usergrid/CoreITSuite.java | 58 ----
.../java/org/apache/usergrid/CoreTestSuite.java | 45 ----
.../apache/usergrid/batch/SchedulerITSuite.java | 50 ----
.../usergrid/batch/SchedulerTestSuite.java | 34 ---
.../batch/job/AbstractSchedulerRuntimeIT.java | 18 +-
.../apache/usergrid/persistence/CounterIT.java | 6 +-
.../apache/usergrid/persistence/IndexIT.java | 21 +-
.../cassandra/EntityManagerFactoryImplIT.java | 21 +-
.../query/AbstractIteratingQueryIT.java | 15 +-
.../system/UsergridSystemMonitorIT.java | 14 +-
.../persistence/index/AliasedEntityIndex.java | 48 ++++
.../usergrid/persistence/index/EntityIndex.java | 5 -
.../usergrid/persistence/index/IndexFig.java | 3 +
.../index/impl/EsEntityIndexBatchImpl.java | 43 ++-
.../index/impl/EsEntityIndexImpl.java | 46 ++--
.../persistence/index/impl/EsIndexCache.java | 114 ++++++++
.../persistence/index/impl/EntityIndexTest.java | 61 ++++-
.../persistence/queue/DefaultQueueManager.java | 68 +++++
stack/loadtests/runtests.sh | 120 +++++++++
.../apache/usergrid/helpers/Extractors.scala | 84 ++++++
.../org/apache/usergrid/helpers/Setup.scala | 30 +--
.../usergrid/scenarios/UserScenarios.scala | 177 +++++++++---
.../org/apache/usergrid/settings/Headers.scala | 6 +-
.../simulations/DeleteUsersSimulation.scala | 56 ++++
.../simulations/GetUserPagesSimulation.scala | 51 ++++
.../simulations/GetUsersSimulation.scala | 51 ++++
.../simulations/PutUsersSimulation.scala | 56 ++++
stack/services/pom.xml | 268 ++++---------------
.../cassandra/ManagementServiceImpl.java | 2 -
.../usergrid/ConcurrentServiceITSuite.java | 60 -----
.../org/apache/usergrid/NewOrgAppAdminRule.java | 159 +++++++++++
.../org/apache/usergrid/ServiceITSuite.java | 64 -----
.../apache/usergrid/management/EmailFlowIT.java | 124 ++++++---
.../usergrid/management/OrganizationIT.java | 36 ++-
.../org/apache/usergrid/management/RoleIT.java | 12 +-
.../cassandra/ApplicationCreatorIT.java | 46 ++--
.../management/cassandra/ExportServiceIT.java | 63 +++--
.../cassandra/ManagementServiceIT.java | 94 ++++---
.../security/providers/FacebookProviderIT.java | 21 +-
.../providers/PingIdentityProviderIT.java | 12 +-
.../security/tokens/TokenServiceIT.java | 55 ++--
.../usergrid/services/AbstractServiceIT.java | 13 +-
.../usergrid/services/ServiceFactoryIT.java | 4 +-
.../usergrid/services/ServiceRequestIT.java | 14 +-
.../usergrid/services/TestQueueManager.java | 65 -----
.../AbstractServiceNotificationIT.java | 16 +-
.../apns/NotificationsServiceIT.java | 12 +-
.../gcm/NotificationsServiceIT.java | 15 +-
.../java/org/apache/usergrid/TestHelper.java | 75 ++++++
56 files changed, 1632 insertions(+), 1128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a8cc5870/stack/core/src/test/java/org/apache/usergrid/persistence/query/AbstractIteratingQueryIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a8cc5870/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a8cc5870/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a8cc5870/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 1ae2d6a,67e1680..b743d83
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -26,30 -23,9 +26,31 @@@ import java.util.ArrayList
import java.util.List;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.index.utils.StringUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.Health;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
- import org.apache.usergrid.persistence.index.EntityIndex;
++import org.apache.usergrid.persistence.index.AliasedEntityIndex;
++import org.apache.usergrid.persistence.index.AliasedEntityIndex.AliasType;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.IndexFig;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.exceptions.IndexException;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
@@@ -86,6 -54,29 +84,8 @@@ import org.elasticsearch.search.sort.So
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.util.Health;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.index.exceptions.IndexException;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
+ import rx.Observable;
+ import rx.functions.Func1;
-import rx.schedulers.Schedulers;
-
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
/**
[04/50] incubator-usergrid git commit: This contains the extra test.
Posted by sf...@apache.org.
This contains the extra test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f5df7b7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f5df7b7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f5df7b7b
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: f5df7b7b0eb9741abfe204c61136d12e7f7ae77c
Parents: 839fc2b
Author: grey <gr...@apigee.com>
Authored: Wed Oct 29 16:32:11 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Wed Oct 29 16:32:11 2014 -0700
----------------------------------------------------------------------
.../impl/EntityVersionCreatedTaskTest.java | 135 +++++++++++++++++++
1 file changed, 135 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f5df7b7b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 4f1abca..9f8d77f 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -3,11 +3,13 @@ package org.apache.usergrid.persistence.collection.impl;
import java.util.ArrayList;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Semaphore;
import org.junit.AfterClass;
import org.junit.Test;
@@ -132,4 +134,137 @@ public class EntityVersionCreatedTaskTest {
}
+ @Test//(timeout=10000)
+ public void oneListener()//why does it matter if it has a version or not without a listener
+ throws ExecutionException, InterruptedException, ConnectionException {
+
+
+ final SerializationFig serializationFig = mock( SerializationFig.class );
+
+ when( serializationFig.getBufferSize() ).thenReturn( 10 );
+
+ final MvccEntitySerializationStrategy ess =
+ mock( MvccEntitySerializationStrategy.class );
+
+ final MvccLogEntrySerializationStrategy less =
+ mock( MvccLogEntrySerializationStrategy.class );
+
+ final Keyspace keyspace = mock( Keyspace.class );
+
+ final MutationBatch entityBatch = mock( MutationBatch.class );
+
+ final MutationBatch logBatch = mock( MutationBatch.class );
+
+ final EntityVersionCreatedFactory entityVersionCreatedFactory = mock(EntityVersionCreatedFactory.class);
+
+ when( keyspace.prepareMutationBatch() )
+ .thenReturn( mock( MutationBatch.class ) ) // don't care what happens to this one
+ .thenReturn( entityBatch )
+ .thenReturn( logBatch );
+
+ // intentionally no events
+
+ //create a latch for the event listener, and add it to the list of events
+ final int sizeToReturn = 1;
+
+ final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+
+ final EntityVersionCreatedTest eventListener = new EntityVersionCreatedTest(latch);
+
+ final Set<EntityVersionCreated> listeners = mock( Set.class );//new HashSet<EntityVersionCreated>();
+
+ final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
+
+
+ when ( listeners.size()).thenReturn( 1 );
+ when ( listeners.iterator()).thenReturn( helper );
+ when ( helper.next() ).thenReturn( eventListener );
+
+ final Id applicationId = new SimpleId( "application" );
+
+ final CollectionScope appScope = new CollectionScopeImpl(
+ applicationId, applicationId, "users" );
+
+ final Id entityId = new SimpleId( "user" );
+
+ final Entity entity = new Entity( entityId );
+
+ final MvccEntity mvccEntity = new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.COMPLETE,entity );
+
+
+ // mock up a single log entry for our first test
+ final LogEntryMock logEntryMock =
+ LogEntryMock.createLogEntryMock(less, appScope, entityId, 2 );
+
+ final UUID version = logEntryMock.getEntries().iterator().next().getVersion();
+
+ final UniqueValueSerializationStrategy uvss =
+ mock( UniqueValueSerializationStrategy.class );
+
+ //when ( listeners.iterator().next().versionCreated(appScope,entity));
+
+
+ EntityVersionCreatedTask entityVersionCreatedTask =
+ new EntityVersionCreatedTask(entityVersionCreatedFactory,
+ serializationFig,
+ less,
+ ess,
+ uvss,
+ keyspace,
+ appScope,
+ listeners,
+ entity);
+
+ // start the task
+ ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
+
+ // wait for the task
+ future.get();
+
+ //mocked listener makes sure that the task is called
+ verify( listeners ).size();
+ verify( listeners ).iterator();
+ verify( helper ).next();
+
+ }
+
+ private static class EntityVersionCreatedTest implements EntityVersionCreated {
+ final CountDownLatch invocationLatch;
+
+ private EntityVersionCreatedTest( final CountDownLatch invocationLatch) {
+ this.invocationLatch = invocationLatch;
+ }
+
+ @Override
+ public void versionCreated( final CollectionScope scope, final Entity entity ) {
+ invocationLatch.countDown();
+ }
+ }
+
+
+// private static class SlowListener extends EntityVersionCreatedTest {
+// final Semaphore blockLatch;
+//
+// private SlowListener( final CountDownLatch invocationLatch, final Semaphore blockLatch ) {
+// super( invocationLatch );
+// this.blockLatch = blockLatch;
+// }
+//
+//
+// @Override
+// public void versionDeleted( final CollectionScope scope, final Id entityId,
+// final List<MvccEntity> entityVersion ) {
+//
+// //wait for unblock to happen before counting down invocation latches
+// try {
+// blockLatch.acquire();
+// }
+// catch ( InterruptedException e ) {
+// throw new RuntimeException( e );
+// }
+// super.versionDeleted( scope, entityId, entityVersion );
+// }
+// }
+
}
[32/50] incubator-usergrid git commit: Merge remote-tracking branch
'origin/two-dot-o' into two-dot-o-events
Posted by sf...@apache.org.
Merge remote-tracking branch 'origin/two-dot-o' into two-dot-o-events
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ee260cc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ee260cc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ee260cc1
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: ee260cc158c0df12198a0671d96e07a0089f17ca
Parents: 43105f1 74866a5
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Nov 24 09:28:17 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Nov 24 09:28:17 2014 -0500
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/GeoIT.java | 404 +++++++++++++------
.../index/impl/EsEntityIndexBatchImpl.java | 69 ++--
.../index/impl/EsEntityIndexImpl.java | 105 +++--
.../persistence/index/impl/EsProvider.java | 55 +--
.../persistence/index/impl/EsQueryVistor.java | 3 +-
.../index/impl/FailureMonitorImpl.java | 4 +-
.../persistence/index/impl/IndexingUtils.java | 51 +--
.../index/query/CandidateResults.java | 3 -
.../persistence/index/query/EntityResults.java | 4 +-
.../persistence/index/query/Identifier.java | 3 +-
.../usergrid/persistence/index/query/Query.java | 26 +-
.../apache/usergrid/rest/AbstractRestIT.java | 5 +-
.../collection/BrowserCompatibilityTest.java | 2 +-
.../collection/DuplicateNameIT.java | 2 +-
.../collection/activities/PutTest.java | 2 +-
.../collection/groups/GroupResourceIT.java | 240 +++++++++--
.../collection/paging/PagingEntitiesTest.java | 4 +-
.../collection/paging/PagingResourceIT.java | 14 +-
.../users/ConnectionResourceTest.java | 16 +-
.../collection/users/OwnershipResourceIT.java | 18 +-
.../collection/users/RetrieveUsersTest.java | 4 +-
.../applications/queries/AndOrQueryTest.java | 13 +-
.../queries/BadGrammarQueryTest.java | 2 +-
.../applications/queries/GeoPagingTest.java | 222 +++++++++-
.../applications/queries/MatrixQueryTests.java | 2 +-
.../rest/applications/queries/OrderByTest.java | 6 +-
.../usergrid/rest/test/resource/Connection.java | 9 +
.../rest/test/resource/TestContext.java | 66 ++-
.../rest/test/resource/ValueResource.java | 2 +
.../rest/test/resource/app/Application.java | 16 +-
.../rest/test/resource/app/Collection.java | 81 ++++
.../usergrid/rest/test/resource/app/Group.java | 56 +++
.../test/resource/app/GroupsCollection.java | 66 +++
.../usergrid/rest/test/resource/app/Role.java | 56 +++
.../rest/test/resource/app/RolesCollection.java | 64 +++
.../rest/test/resource/app/UsersCollection.java | 26 +-
36 files changed, 1336 insertions(+), 385 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee260cc1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index e0ec1ed,4ce4d12..6e68791
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@@ -173,9 -179,10 +185,10 @@@ public class EsEntityIndexBatchImpl imp
}
- log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
+ log.debug( "De-indexing type {} with documentId '{}'", entityType, indexId );
- bulkRequest.add( client.prepareDelete( indexName, entityType, indexId ).setRefresh( refresh ) );
+ bulkRequest.add( client.prepareDelete(
+ indexName, entityType, indexId ).setRefresh( refresh ) );
log.debug( "Deindexed Entity with index id " + indexId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee260cc1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 406c967,22aa05c..53aaf54
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -221,16 -214,16 +222,16 @@@ public class EsEntityIndexImpl implemen
*/
private void createMappings() throws IOException {
- XContentBuilder xcb =
- IndexingUtils.createDoubleStringIndexMapping( XContentFactory.jsonBuilder(), "_default_" );
+ XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
+ XContentFactory.jsonBuilder(), "_default_" );
- PutIndexTemplateResponse pitr =
- esProvider.getClient().admin().indices().preparePutTemplate( "usergrid_template" )
- .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_",
- xcb ) // set mapping as the default for all types
- .execute().actionGet();
+ PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
+ .preparePutTemplate( "usergrid_template" )
+ // set mapping as the default for all types
+ .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_", xcb )
+ .execute().actionGet();
- if(!pitr.isAcknowledged()){
+ if ( !pitr.isAcknowledged() ) {
throw new IndexException( "Unable to create default mappings" );
}
}
@@@ -306,10 -300,10 +308,9 @@@
}
-
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
- this.indexName, context, entityTypes, srb
- } );
+ logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ",
+ new Object[] { this.indexName, context, entityTypes, srb } );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ee260cc1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index ee5557c,6d34b15..4319ce4
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@@ -158,11 -153,11 +155,11 @@@ public class IndexingUtils
.startArray( "dynamic_templates" )
- //we need most specific mappings first since it's a stop on match algorithm
+ // we need most specific mappings first since it's a stop on match algorithm
.startObject()
- .startObject( "context_template" )
- .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
+ .startObject( "entity_id_template" )
+ .field( "match", IndexingUtils.ENTITY_ID_FIELDNAME )
.field( "match_mapping_type", "string" )
.startObject( "mapping" ).field( "type", "string" )
.field( "index", "not_analyzed" )
@@@ -170,47 -165,44 +167,53 @@@
.endObject()
.endObject()
-
.startObject()
- .startObject( "context_template" )
+ .startObject( "entity_context_template" )
.field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
.field( "match_mapping_type", "string" )
- .startObject( "mapping" ).field( "type", "string" )
+ .startObject( "mapping" )
+ .field( "type", "string" )
.field( "index", "not_analyzed" ).endObject()
.endObject()
.endObject()
+ .startObject()
+ .startObject( "entity_version_template" )
+ .field( "match", IndexingUtils.ENTITY_VERSION_FIELDNAME )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" ).field( "type", "long" )
+ .endObject()
+ .endObject()
+ .endObject()
+
// any string with field name that starts with sa_ gets analyzed
.startObject()
- .startObject( "template_1" ).field( "match", ANALYZED_STRING_PREFIX + "*" )
- .field( "match_mapping_type", "string" ).startObject( "mapping" ).field( "type", "string" )
- .field( "index", "analyzed" ).endObject().endObject()
+ .startObject( "template_1" )
+ .field( "match", ANALYZED_STRING_PREFIX + "*" )
+ .field( "match_mapping_type", "string" ).startObject( "mapping" )
+ .field( "type", "string" )
+ .field( "index", "analyzed" )
+ .endObject()
.endObject()
+ .endObject()
- // all other strings are not analyzed
- .startObject()
- .startObject( "template_2" )
- //todo, should be string prefix, remove 2 field mapping
- .field( "match", "*" ).field( "match_mapping_type", "string" ).startObject( "mapping" )
+ // all other strings are not analyzed
+ .startObject()
+ .startObject( "template_2" )
+ //todo, should be string prefix, remove 2 field mapping
+ .field( "match", "*" )
+ .field( "match_mapping_type", "string" )
+ .startObject( "mapping" )
.field( "type", "string" )
.field( "index", "not_analyzed" )
- .endObject()
.endObject()
.endObject()
+ .endObject()
- // fields names starting with go_ get geo-indexed
- .startObject()
- .startObject( "template_3" )
- .field( "match", GEO_PREFIX + "location" )
+ // fields names starting with go_ get geo-indexed
+ .startObject()
+ .startObject( "template_3" )
+ .field( "match", GEO_PREFIX + "location" )
.startObject( "mapping" )
.field( "type", "geo_point" )
.endObject()
[10/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ac03027c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ac03027c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ac03027c
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: ac03027c0695eca8f955fa787f3f07842ac0b727
Parents: ec0d69a 0bfca7e
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 30 09:50:57 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 30 09:50:57 2014 -0400
----------------------------------------------------------------------
portal/js/push/push-config.html | 89 ++++++++++----------
.../services/notifications/gcm/GCMAdapter.java | 10 ++-
2 files changed, 54 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
[05/50] incubator-usergrid git commit: Contains code that helps make
it easier to mock.
Posted by sf...@apache.org.
Contains code that helps make it easier to mock.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/da38f0bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/da38f0bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/da38f0bf
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: da38f0bfebe87d1e0fb7fcff85a55f2576762302
Parents: f5df7b7
Author: grey <gr...@apigee.com>
Authored: Wed Oct 29 16:33:00 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Wed Oct 29 16:33:00 2014 -0700
----------------------------------------------------------------------
.../persistence/collection/impl/EntityVersionCreatedTask.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/da38f0bf/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
index b78b09c..fdfb551 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
@@ -122,7 +122,10 @@ public class EntityVersionCreatedTask implements Task<Void> {
}
if ( listenerSize == 1 ) {
- listeners.iterator().next().versionCreated( collectionScope,entity );
+ EntityVersionCreated entityVersionCreated = listeners.iterator().next();
+
+ entityVersionCreated.versionCreated( collectionScope,entity );
+
return;
}
[09/50] incubator-usergrid git commit: Minor refactoring,
renamed our event handlers to use suffix "Handler", fixed some headers,
imports, etc.
Posted by sf...@apache.org.
Minor refactoring, renamed our event handlers to use suffix "Handler", fixed some headers, imports, etc.
This closes #91
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ec0d69aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ec0d69aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ec0d69aa
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: ec0d69aa32d952c80a9152b22fe9f3de2bade08c
Parents: f813136
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 30 09:49:51 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 30 09:49:51 2014 -0400
----------------------------------------------------------------------
.../usergrid/corepersistence/GuiceModule.java | 8 +-
.../events/EntityDeletedHandler.java | 63 +++++++++++++++
.../events/EntityDeletedImpl.java | 64 ---------------
.../events/EntityVersionCreatedHandler.java | 62 +++++++++++++++
.../events/EntityVersionCreatedImpl.java | 49 ------------
.../events/EntityVersionDeletedHandler.java | 84 ++++++++++++++++++++
.../events/EntityVersionDeletedImpl.java | 83 -------------------
.../corepersistence/StaleIndexCleanupTest.java | 14 ++--
.../impl/EntityVersionCleanupTask.java | 2 +-
9 files changed, 222 insertions(+), 207 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec0d69aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index a11fe8b..21eba41 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@ -17,8 +17,8 @@ package org.apache.usergrid.corepersistence;
import com.google.inject.AbstractModule;
import com.google.inject.multibindings.Multibinder;
-import org.apache.usergrid.corepersistence.events.EntityDeletedImpl;
-import org.apache.usergrid.corepersistence.events.EntityVersionDeletedImpl;
+import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
+import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
@@ -50,11 +50,11 @@ public class GuiceModule extends AbstractModule {
Multibinder<EntityDeleted> entityBinder
= Multibinder.newSetBinder(binder(), EntityDeleted.class);
- entityBinder.addBinding().to(EntityDeletedImpl.class);
+ entityBinder.addBinding().to(EntityDeletedHandler.class);
Multibinder<EntityVersionDeleted> versionBinder
= Multibinder.newSetBinder(binder(), EntityVersionDeleted.class);
- versionBinder.addBinding().to(EntityVersionDeletedImpl.class);
+ versionBinder.addBinding().to(EntityVersionDeletedHandler.class);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec0d69aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
new file mode 100644
index 0000000..b5cb1ff
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.corepersistence.events;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.event.EntityDeleted;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import java.util.UUID;
+import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
+import org.apache.usergrid.corepersistence.CpSetup;
+import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Delete all Query Index indexes associated with an Entity that has just been deleted.
+ */
+public class EntityDeletedHandler implements EntityDeleted {
+ private static final Logger logger = LoggerFactory.getLogger(EntityDeletedHandler.class );
+
+
+ public EntityDeletedHandler() {
+ logger.debug("Created");
+ }
+
+ @Override
+ public void deleted(CollectionScope scope, Id entityId, UUID version) {
+
+ logger.debug("Entering deleted for entity {}:{} v {} "
+ + "scope\n name: {}\n owner: {}\n app: {}",
+ new Object[] { entityId.getType(), entityId.getUuid(), version,
+ scope.getName(), scope.getOwner(), scope.getApplication()});
+
+ HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
+
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+
+ EntityIndexBatch batch = ei.createBatch();
+
+ batch.deleteEntity( entityId );
+ batch.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec0d69aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
deleted file mode 100644
index 925e01d..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-
-package org.apache.usergrid.corepersistence.events;
-
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.event.EntityDeleted;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import java.util.UUID;
-import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.CpSetup;
-import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Delete all Query Index indexes associated with an Entity that has just been deleted.
- */
-public class EntityDeletedImpl implements EntityDeleted {
- private static final Logger logger = LoggerFactory.getLogger( EntityDeletedImpl.class );
-
-
- public EntityDeletedImpl() {
- logger.debug("Created");
- }
-
- @Override
- public void deleted(CollectionScope scope, Id entityId, UUID version) {
-
- logger.debug("Entering deleted for entity {}:{} v {} "
- + "scope\n name: {}\n owner: {}\n app: {}",
- new Object[] { entityId.getType(), entityId.getUuid(), version,
- scope.getName(), scope.getOwner(), scope.getApplication()});
-
- HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
-
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
-
- EntityIndexBatch batch = ei.createBatch();
-
- batch.deleteEntity( entityId );
- batch.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec0d69aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
new file mode 100644
index 0000000..64d1125
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.corepersistence.events;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
+import org.apache.usergrid.corepersistence.CpSetup;
+import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+
+/**
+ * Clean up stale entity indexes when new version of Entity created. Called when an Entity is
+ * updated by the Collections module and we react by calling the Query Index module and removing
+ * any indexes that exist for previous versions of the the Entity.
+ */
+public class EntityVersionCreatedHandler implements EntityVersionCreated {
+
+ private static final Logger logger = LoggerFactory.getLogger(EntityVersionCreatedHandler.class );
+
+ public EntityVersionCreatedHandler() {
+ logger.debug("EntityVersionCreated");
+ }
+
+ @Override
+ public void versionCreated( final CollectionScope scope, final Entity entity ) {
+ logger.debug("Entering deleted for entity {}:{} v {} "
+ + "scope\n name: {}\n owner: {}\n app: {}",
+ new Object[] { entity.getId().getType(), entity.getId().getUuid(), entity.getVersion(),
+ scope.getName(), scope.getOwner(), scope.getApplication()});
+
+ HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
+
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+
+ EntityIndexBatch batch = ei.createBatch();
+
+ batch.deindexPreviousVersions( entity );
+ batch.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec0d69aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedImpl.java
deleted file mode 100644
index c7aada7..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedImpl.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.apache.usergrid.corepersistence.events;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.CpEntityManager;
-import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.CpSetup;
-import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.model.entity.Entity;
-
-
-/**
- * Created by ApigeeCorporation on 10/24/14.
- */
-public class EntityVersionCreatedImpl implements EntityVersionCreated{
-
- private static final Logger logger = LoggerFactory.getLogger( EntityVersionCreatedImpl.class );
-
- public EntityVersionCreatedImpl() {
- logger.debug("EntityVersionCreated");
- }
-
- @Override
- public void versionCreated( final CollectionScope scope, final Entity entity ) {
- logger.debug("Entering deleted for entity {}:{} v {} "
- + "scope\n name: {}\n owner: {}\n app: {}",
- new Object[] { entity.getId().getType(), entity.getId().getUuid(), entity.getVersion(),
- scope.getName(), scope.getOwner(), scope.getApplication()});
-
- HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
-
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
-
- EntityIndexBatch batch = ei.createBatch();
-
- batch.deindexPreviousVersions( entity );
- batch.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec0d69aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
new file mode 100644
index 0000000..046e9c2
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.corepersistence.events;
+
+import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
+import org.apache.usergrid.corepersistence.CpSetup;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import rx.functions.Func1;
+import rx.schedulers.Schedulers;
+
+import java.util.List;
+
+import com.google.inject.Inject;
+import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
+
+
+/**
+ * Remove Entity index when specific version of Entity is deleted.
+ * TODO: do we need this? Don't our version-created and entity-deleted handlers take care of this?
+ * If we do need it then it should be wired in via GuiceModule in the corepersistence package.
+ */
+public class EntityVersionDeletedHandler implements EntityVersionDeleted {
+
+ private final SerializationFig serializationFig;
+
+ @Inject
+ public EntityVersionDeletedHandler(SerializationFig fig) {
+ this.serializationFig = fig;
+ }
+
+ @Override
+ public void versionDeleted(
+ final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
+
+ HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
+
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+
+ final EntityIndexBatch eibatch = ei.createBatch();
+
+ final IndexScope indexScope = new IndexScopeImpl(
+ new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
+ scope.getName()
+ );
+ rx.Observable.from(entityVersions)
+ .subscribeOn(Schedulers.io())
+ .buffer(serializationFig.getBufferSize())
+ .map(new Func1<List<MvccEntity>, List<MvccEntity>>() {
+ @Override
+ public List<MvccEntity> call(List<MvccEntity> entityList) {
+ for (MvccEntity entity : entityList) {
+ eibatch.deindex(indexScope, entityId, entity.getVersion());
+ }
+ eibatch.execute();
+ return entityList;
+ }
+ }).toBlocking().last();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec0d69aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
deleted file mode 100644
index fb52573..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.corepersistence.events;
-
-import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.CpSetup;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-import rx.functions.Func1;
-import rx.schedulers.Schedulers;
-
-import java.util.List;
-
-import com.google.inject.Inject;
-import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
-
-
-/**
- * Purge old entity versions
- */
-public class EntityVersionDeletedImpl implements EntityVersionDeleted {
-
- private final SerializationFig serializationFig;
-
- @Inject
- public EntityVersionDeletedImpl(SerializationFig fig) {
- this.serializationFig = fig;
- }
-
- @Override
- public void versionDeleted(
- final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
-
- HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
-
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
-
- final EntityIndexBatch eibatch = ei.createBatch();
-
- final IndexScope indexScope = new IndexScopeImpl(
- new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
- scope.getName()
- );
- rx.Observable.from(entityVersions)
- .subscribeOn(Schedulers.io())
- .buffer(serializationFig.getBufferSize())
- .map(new Func1<List<MvccEntity>, List<MvccEntity>>() {
- @Override
- public List<MvccEntity> call(List<MvccEntity> entityList) {
- for (MvccEntity entity : entityList) {
- eibatch.deindex(indexScope, entityId, entity.getVersion());
- }
- eibatch.execute();
- return entityList;
- }
- }).toBlocking().last();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec0d69aa/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index c4b362d..d828733 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -1,17 +1,19 @@
/*
- * Copyright 2014 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
* You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
- * limitations under the License.
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
*/
package org.apache.usergrid.corepersistence;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec0d69aa/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index d45eeb9..d4bf6e6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -1,6 +1,6 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
+ * contributor license agreements. The ASF licenses this file to You
* under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License.
* You may obtain a copy of the License at
[43/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bd3b422d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bd3b422d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bd3b422d
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: bd3b422da1b7a605c2aedd31245379b8dbbbf845
Parents: ed0ea46 e29a99e
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Dec 8 15:53:16 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Dec 8 15:53:16 2014 -0500
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 216 +++---
.../corepersistence/util/CpEntityMapUtils.java | 32 +-
.../batch/job/SchedulerRuntimeIntervalIT.java | 49 +-
.../migration/EntityDataMigrationIT.java | 139 ++--
.../migration/EntityTypeMappingMigrationIT.java | 89 ++-
.../apache/usergrid/persistence/CounterIT.java | 89 ++-
.../usergrid/persistence/CountingMutatorIT.java | 4 +-
.../persistence/EntityDictionaryIT.java | 11 +-
.../usergrid/persistence/EntityManagerIT.java | 47 +-
.../org/apache/usergrid/persistence/GeoIT.java | 771 +++++++++----------
.../persistence/GeoQueryBooleanTest.java | 7 +-
.../apache/usergrid/persistence/IndexIT.java | 14 +-
.../usergrid/persistence/PathQueryIT.java | 10 +-
.../PerformanceEntityRebuildIndexTest.java | 6 +-
.../usergrid/persistence/PermissionsIT.java | 5 +-
.../cassandra/EntityManagerFactoryImplIT.java | 9 +-
.../query/AbstractIteratingQueryIT.java | 3 +-
stack/core/src/test/resources/log4j.properties | 5 +-
.../data/DataMigrationManagerImpl.java | 29 +-
.../persistence/index/IndexIdentifier.java | 4 +
.../index/impl/EsEntityIndexImpl.java | 58 +-
21 files changed, 836 insertions(+), 761 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bd3b422d/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bd3b422d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 86826a5,eb050d6..0614220
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -351,9 -347,10 +356,9 @@@ public class EsEntityIndexImpl implemen
}
-
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
- this.alias, context, entityTypes, srb
+ logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ",
+ new Object[] { this.alias, context, entityTypes, srb
} );
}
@@@ -483,9 -480,13 +488,16 @@@
final QueryBuilder queryBuilder =
QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+
+ final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
+ .setTypes(searchTypes.getTypeNames())
+ .setScroll(cursorTimeout + "m")
+ .setQuery(queryBuilder);
+
+
+ final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
+ .setTypes(searchTypes.getTypeNames())
+ .setScroll(cursorTimeout + "m").setQuery(queryBuilder);
final SearchResponse searchResponse;
try {
[26/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bce81429
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bce81429
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bce81429
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: bce814297beb3fd4afcd7e89f8edebede8ac330f
Parents: e826cc2 b7f81ef
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Nov 7 10:01:07 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Nov 7 10:01:07 2014 -0500
----------------------------------------------------------------------
portal/config.js | 3 +-
stack/awscluster/assembly.xml | 8 +-
stack/awscluster/pom.xml | 379 ++++++-----
.../main/dist/init_instance/create_keyspaces.sh | 2 +
.../src/main/dist/init_instance/create_raid0.sh | 9 +-
.../main/dist/init_instance/init_db_server.sh | 15 +-
.../main/dist/init_instance/init_es_server.sh | 8 +-
.../dist/init_instance/init_graphite_server.sh | 2 +
.../dist/init_instance/init_opscenter_server.sh | 287 +++++++++
.../main/dist/init_instance/init_rest_server.sh | 66 +-
.../dist/init_instance/install_cassandra.sh | 22 +-
.../dist/init_instance/install_elasticsearch.sh | 57 +-
.../init_instance/install_opscenter_agent.sh | 12 +-
stack/awscluster/src/main/dist/update.sh | 16 +-
.../main/groovy/configure_elasticsearch.groovy | 84 ++-
.../groovy/configure_opscenter_agent.groovy | 4 +-
.../groovy/configure_opscenter_cassandra.groovy | 120 ++++
.../groovy/configure_opscenter_usergrid.groovy | 58 ++
.../src/main/groovy/configure_usergrid.groovy | 19 +-
stack/awscluster/ugcluster-cf.json | 321 +++++++++-
.../main/resources/usergrid-default.properties | 5 +-
stack/core/pom.xml | 16 +-
.../corepersistence/CpEntityManager.java | 5 +-
.../corepersistence/CpEntityManagerFactory.java | 22 +
.../corepersistence/CpManagerCache.java | 12 +-
.../corepersistence/CpRelationManager.java | 52 +-
.../usergrid/corepersistence/CpSetup.java | 5 +-
.../usergrid/corepersistence/GuiceModule.java | 6 +
.../HybridEntityManagerFactory.java | 19 +
.../migration/GraphShardVersionMigration.java | 43 ++
.../corepersistence/migration/Versions.java | 37 ++
.../persistence/EntityManagerFactory.java | 18 +
.../cassandra/EntityManagerFactoryImpl.java | 19 +
.../usergrid/ConcurrentCoreIteratorITSuite.java | 3 +-
.../usergrid/ConcurrentCoreTestSuite.java | 4 +-
.../org/apache/usergrid/CoreITSetupImpl.java | 4 +-
.../java/org/apache/usergrid/CoreITSuite.java | 5 +-
.../java/org/apache/usergrid/CoreTestSuite.java | 4 +-
.../batch/job/AbstractSchedulerRuntimeIT.java | 9 +-
.../SimpleIndexBucketLocatorImplTest.java | 177 ------
.../SimpleIndexShardLocatorImplTest.java | 177 ++++++
stack/corepersistence/collection/pom.xml | 7 +-
.../impl/CollectionPrefixedKey.java | 100 +++
.../impl/CollectionScopedRowKeySerializer.java | 25 +-
.../MvccEntitySerializationStrategyImpl.java | 88 ++-
.../MvccLogEntrySerializationStrategyImpl.java | 66 +-
.../serialization/impl/SerializationModule.java | 2 +-
.../UniqueValueSerializationStrategyImpl.java | 31 +-
.../collection/CollectionContextImplTest.java | 3 -
.../EntityCollectionManagerFactoryTest.java | 6 +-
.../collection/EntityCollectionManagerIT.java | 4 +-
.../EntityCollectionManagerStressTest.java | 6 +-
.../EntityCollectionManagerSyncIT.java | 4 +-
.../collection/guice/MigrationManagerRule.java | 4 +-
.../collection/guice/TestCollectionModule.java | 3 +-
.../collection/guice/TestModule.java | 48 --
...niqueValueSerializationStrategyImplTest.java | 4 +-
.../stage/write/WriteOptimisticVerifyTest.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 4 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 4 +-
...MvccEntitySerializationStrategyImplTest.java | 14 +-
.../impl/MvccLESSTransientTest.java | 78 ++-
...ccLogEntrySerializationStrategyImplTest.java | 4 +-
.../impl/ScopedRowKeySerializerTest.java | 18 +-
stack/corepersistence/common/pom.xml | 15 +-
.../core/astyanax/BucketScopedRowKey.java | 114 ++++
.../astyanax/BucketScopedRowKeySerializer.java | 91 +++
.../core/astyanax/CassandraConfig.java | 6 +
.../core/astyanax/CassandraConfigImpl.java | 23 +
.../persistence/core/astyanax/CassandraFig.java | 14 +
.../persistence/core/astyanax/ColumnSearch.java | 13 +-
.../core/astyanax/CompositeFieldSerializer.java | 1 +
.../core/astyanax/IdRowCompositeSerializer.java | 1 +
.../core/astyanax/MultiRowColumnIterator.java | 236 +++++--
.../core/astyanax/MultiTennantColumnFamily.java | 8 +-
.../OrganizationScopedRowKeySerializer.java | 85 ---
.../persistence/core/astyanax/ScopedRowKey.java | 13 +-
.../core/astyanax/ScopedRowKeySerializer.java | 83 +++
.../astyanax/StringRowCompositeSerializer.java | 68 ++
.../persistence/core/guice/CommonModule.java | 43 +-
.../persistence/core/guice/CurrentImpl.java | 42 ++
.../persistence/core/guice/PreviousImpl.java | 42 ++
.../persistence/core/guice/ProxyImpl.java | 42 ++
.../persistence/core/migration/Migration.java | 36 --
.../core/migration/MigrationException.java | 37 --
.../core/migration/MigrationManager.java | 34 -
.../core/migration/MigrationManagerFig.java | 22 -
.../core/migration/MigrationManagerImpl.java | 196 ------
.../core/migration/data/DataMigration.java | 84 +++
.../migration/data/DataMigrationException.java | 39 ++
.../migration/data/DataMigrationManager.java | 55 ++
.../data/DataMigrationManagerImpl.java | 258 ++++++++
.../data/MigrationInfoSerialization.java | 66 ++
.../data/MigrationInfoSerializationImpl.java | 178 ++++++
.../core/migration/schema/Migration.java | 36 ++
.../migration/schema/MigrationException.java | 37 ++
.../core/migration/schema/MigrationManager.java | 34 +
.../migration/schema/MigrationManagerFig.java | 22 +
.../migration/schema/MigrationManagerImpl.java | 196 ++++++
.../core/shard/ExpandingShardLocator.java | 95 +++
.../persistence/core/shard/ShardLocator.java | 77 +++
.../persistence/core/shard/StringHashUtils.java | 54 ++
.../core/astyanax/ColumnNameIteratorTest.java | 6 +
.../MultiKeyColumnNameIteratorTest.java | 6 +
.../astyanax/MultiRowColumnIteratorTest.java | 42 ++
.../core/cassandra/CassandraRule.java | 2 +-
.../persistence/core/cassandra/ITRunner.java | 60 --
.../core/guice/MigrationManagerRule.java | 4 +-
.../core/guice/TestCommonModule.java | 34 +
.../persistence/core/guice/TestModule.java | 48 ++
.../data/DataMigrationManagerImplTest.java | 220 +++++++
.../data/MigrationInfoSerializationTest.java | 105 ++++
.../core/shard/ExpandingShardLocatorTest.java | 65 ++
.../core/shard/ShardLocatorTest.java | 64 ++
.../persistence/core/test/ITRunner.java | 119 ++++
.../persistence/core/test/UseModules.java | 69 ++
stack/corepersistence/graph/pom.xml | 15 +-
.../persistence/graph/guice/GraphModule.java | 34 +-
.../graph/impl/GraphManagerImpl.java | 13 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 3 +-
.../impl/stage/NodeDeleteListenerImpl.java | 3 +-
.../EdgeMetadataSerialization.java | 2 +-
.../graph/serialization/NodeSerialization.java | 2 +-
.../impl/EdgeMetadataSerializationImpl.java | 476 --------------
.../EdgeMetadataSerializationProxyImpl.java | 276 ++++++++
.../impl/EdgeMetadataSerializationV1Impl.java | 478 ++++++++++++++
.../impl/EdgeMetadataSerializationV2Impl.java | 627 +++++++++++++++++++
.../impl/NodeSerializationImpl.java | 36 +-
.../impl/shard/EdgeColumnFamilies.java | 14 +-
.../impl/shard/EdgeShardSerialization.java | 2 +-
.../count/NodeShardCounterSerialization.java | 2 +-
.../NodeShardCounterSerializationImpl.java | 10 +-
.../impl/shard/impl/EdgeSearcher.java | 18 +-
.../shard/impl/EdgeShardSerializationImpl.java | 14 +-
.../impl/ShardedEdgeSerializationImpl.java | 76 +--
.../impl/shard/impl/ShardsColumnIterator.java | 21 +-
.../shard/impl/SizebasedEdgeColumnFamilies.java | 34 +-
.../impl/shard/impl/SourceEdgeSearcher.java | 22 -
.../serialize/EdgeShardRowKeySerializer.java | 2 +
.../shard/impl/serialize/RowSerializer.java | 2 +-
.../shard/impl/serialize/RowTypeSerializer.java | 3 +-
.../graph/CommittedGraphManagerIT.java | 4 +-
.../persistence/graph/GraphManagerIT.java | 9 +-
.../persistence/graph/GraphManagerLoadTest.java | 4 +-
.../graph/GraphManagerShardConsistencyIT.java | 5 +-
.../graph/GraphManagerShardingIT.java | 4 +-
.../graph/GraphManagerStressTest.java | 4 +-
.../usergrid/persistence/graph/SimpleTest.java | 4 +-
.../graph/StorageGraphManagerIT.java | 4 +-
.../graph/guice/TestGraphModule.java | 2 +-
.../graph/impl/EdgeDeleteListenerTest.java | 6 +-
.../graph/impl/NodeDeleteListenerTest.java | 6 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 4 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 6 +-
.../EdgeMetaDataSerializationProxyV1Test.java | 85 +++
.../EdgeMetaDataSerializationProxyV2Test.java | 83 +++
.../EdgeMetaDataSerializationV1Test.java | 69 ++
.../EdgeMetaDataSerializationV2Test.java | 52 ++
.../EdgeMetadataSerializationTest.java | 33 +-
.../EdgeSerializationChopTest.java | 4 +-
.../serialization/EdgeSerializationTest.java | 50 +-
.../serialization/NodeSerializationTest.java | 10 +-
.../PermanentSerializationTest.java | 9 +-
.../graph/serialization/TestCount.java | 52 +-
.../impl/shard/EdgeShardSerializationTest.java | 35 +-
.../impl/shard/NodeShardAllocationTest.java | 15 +-
.../impl/shard/NodeShardCacheTest.java | 118 ++--
.../impl/shard/ShardEntryGroupTest.java | 57 +-
.../impl/shard/ShardGroupCompactionTest.java | 181 +++---
.../shard/count/NodeShardApproximationTest.java | 18 +-
.../NodeShardCounterSerializationTest.java | 10 +-
.../shard/impl/ShardEntryGroupIteratorTest.java | 9 +-
...rceDirectedEdgeDescendingComparatorTest.java | 4 +-
.../1.4-UG/_maven.repositories | 4 -
.../1.4-UG/jukito-build-tools-1.4-UG.jar | Bin 4603 -> 0 bytes
.../1.4-UG/jukito-build-tools-1.4-UG.pom | 13 -
.../jukito-build-tools/maven-metadata-local.xml | 12 -
.../jukito-parent/1.4-UG/_maven.repositories | 3 -
.../1.4-UG/jukito-parent-1.4-UG.pom | 313 ---------
.../jukito-parent/maven-metadata-local.xml | 12 -
.../jukito-samples/1.4-UG/_maven.repositories | 4 -
.../1.4-UG/jukito-samples-1.4-UG.jar | Bin 5103 -> 0 bytes
.../1.4-UG/jukito-samples-1.4-UG.pom | 33 -
.../jukito-samples/maven-metadata-local.xml | 12 -
.../jukito/jukito/1.4-UG/_maven.repositories | 5 -
.../jukito/1.4-UG/jukito-1.4-UG-sources.jar | Bin 30660 -> 0 bytes
.../org/jukito/jukito/1.4-UG/jukito-1.4-UG.jar | Bin 42832 -> 0 bytes
.../org/jukito/jukito/1.4-UG/jukito-1.4-UG.pom | 93 ---
.../org/jukito/jukito/maven-metadata-local.xml | 12 -
stack/corepersistence/map/pom.xml | 6 -
.../persistence/map/guice/MapModule.java | 2 +-
.../persistence/map/impl/MapSerialization.java | 2 +-
.../map/impl/MapSerializationImpl.java | 97 ++-
.../persistence/map/MapManagerTest.java | 4 +-
.../persistence/map/guice/TestMapModule.java | 2 +-
stack/corepersistence/pom.xml | 4 +-
stack/corepersistence/queryindex/pom.xml | 11 -
.../usergrid/persistence/index/IndexFig.java | 14 +-
.../index/impl/EsEntityIndexBatchImpl.java | 23 +-
.../index/impl/EsEntityIndexImpl.java | 109 ++--
.../persistence/index/impl/EsProvider.java | 235 +++----
.../persistence/index/impl/FailureMonitor.java | 41 ++
.../index/impl/FailureMonitorImpl.java | 101 +++
.../index/guice/TestIndexModule.java | 2 +-
.../index/impl/CorePerformanceIT.java | 2 +-
.../index/impl/ElasticSearchResource.java | 171 +++++
.../index/impl/ElasticSearchRule.java | 50 --
.../impl/EntityConnectionIndexImplTest.java | 7 +-
.../persistence/index/impl/EntityIndexTest.java | 10 +-
.../persistence/index/impl/EsRunner.java | 51 ++
.../src/test/resources/usergrid-UNIT.properties | 2 +-
stack/corepersistence/queue/pom.xml | 9 -
.../usergrid/persistence/queue/QueueFig.java | 4 +-
.../persistence/queue/QueueManagerTest.java | 6 +-
.../queue/guice/TestQueueModule.java | 2 +-
stack/loadtests/README.md | 27 +
stack/loadtests/pom.xml | 18 +-
.../datagenerators/EntityDataGenerator.scala | 26 +-
.../datagenerators/FeederGenerator.scala | 15 +-
.../org/apache/usergrid/helpers/Setup.scala | 168 +++++
.../scenarios/ApplicationScenarios.scala | 10 +-
.../scenarios/ConnectionScenarios.scala | 29 +-
.../usergrid/scenarios/DeviceScenarios.scala | 8 +-
.../usergrid/scenarios/EntityScenarios.scala | 65 ++
.../usergrid/scenarios/GeoScenarios.scala | 6 +-
.../scenarios/NotificationScenarios.scala | 25 +-
.../usergrid/scenarios/NotifierScenarios.scala | 20 +-
.../scenarios/OrganizationScenarios.scala | 27 +-
.../usergrid/scenarios/TokenScenarios.scala | 29 +-
.../usergrid/scenarios/UserScenarios.scala | 28 +-
.../org/apache/usergrid/settings/Settings.scala | 32 +-
.../usergrid/simulations/AppSimulation.scala | 52 ++
.../simulations/ConnectionsSimulation.scala | 53 ++
.../PostCustomEntitySimulation.scala | 61 ++
.../simulations/PostDevicesSimulation.scala | 45 --
...PushNotificationTargetDeviceSimulation.scala | 57 --
.../PushNotificationTargetUserSimulation.scala | 59 +-
.../src/test/resources/logback-test.xml | 1 -
stack/loadtests/src/test/scala/Engine.scala | 4 +-
stack/pom.xml | 25 +-
.../apache/usergrid/rest/SystemResource.java | 94 ++-
.../java/org/apache/usergrid/rest/ITSetup.java | 8 +-
.../org/apache/usergrid/rest/RestITSuite.java | 4 +-
.../org/apache/usergrid/ServiceITSetupImpl.java | 1 +
.../org/apache/usergrid/ServiceITSuite.java | 3 +-
stack/test-utils/pom.xml | 7 +
.../org/apache/usergrid/ElasticSearchMain.java | 107 ----
.../apache/usergrid/ElasticSearchResource.java | 178 ------
248 files changed, 8320 insertions(+), 3595 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bce81429/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bce81429/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bce81429/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index e63f272,fb4800c..4268478
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@@ -18,14 -19,11 +18,16 @@@ package org.apache.usergrid.corepersist
import com.google.inject.AbstractModule;
import com.google.inject.multibindings.Multibinder;
+ import org.apache.usergrid.corepersistence.migration.GraphShardVersionMigration;
+import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
+import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
+import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+import org.apache.usergrid.persistence.collection.event.EntityDeleted;
+import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
+import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
+ import org.apache.usergrid.persistence.core.migration.data.DataMigration;
import org.apache.usergrid.persistence.graph.guice.GraphModule;
import org.apache.usergrid.persistence.index.guice.IndexModule;
import org.apache.usergrid.persistence.map.guice.MapModule;
@@@ -51,18 -48,13 +53,22 @@@ public class GuiceModule extends Abstra
install(new MapModule());
install(new QueueModule());
- bind(CpEntityDeleteListener.class).asEagerSingleton();
- bind(CpEntityIndexDeleteListener.class).asEagerSingleton();
+ Multibinder<EntityDeleted> entityBinder
+ = Multibinder.newSetBinder(binder(), EntityDeleted.class);
+ entityBinder.addBinding().to(EntityDeletedHandler.class);
+
+ Multibinder<EntityVersionDeleted> versionBinder
+ = Multibinder.newSetBinder(binder(), EntityVersionDeleted.class);
+ versionBinder.addBinding().to(EntityVersionDeletedHandler.class);
+
+ Multibinder<EntityVersionCreated> versionCreatedMultibinder
+ = Multibinder.newSetBinder( binder(),EntityVersionCreated.class );
+ versionCreatedMultibinder.addBinding().to(EntityVersionCreatedHandler.class);
+ Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
+ dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
+
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bce81429/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bce81429/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
[41/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/532a02cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/532a02cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/532a02cb
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 532a02cb3e4bfecaa91f4174d764d4e8e24fb235
Parents: 21c93f1 6210092
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Dec 3 14:55:25 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Dec 3 14:55:25 2014 -0500
----------------------------------------------------------------------
stack/awscluster/gatling-cluster-cf.json | 387 +++++++++++++++++++
stack/awscluster/pom.xml | 2 +-
.../main/dist/init_instance/install_yourkit.sh | 6 +-
.../src/main/groovy/NodeRegistry.groovy | 197 +++++++---
.../main/groovy/configure_elasticsearch.groovy | 31 +-
.../src/main/groovy/registry_list.groovy | 36 --
stack/awscluster/ugcluster-cf.json | 7 +-
.../MvccEntitySerializationStrategyImpl.java | 2 +-
.../core/astyanax/BucketScopedRowKey.java | 3 +-
.../core/astyanax/CassandraConfig.java | 18 +
.../core/astyanax/ColumnNameIterator.java | 1 -
.../persistence/core/astyanax/ColumnSearch.java | 30 +-
.../persistence/core/astyanax/ColumnTypes.java | 19 +-
.../core/astyanax/FieldBufferBuilder.java | 1 -
.../astyanax/MultiKeyColumnNameIterator.java | 2 -
.../core/astyanax/MultiRowColumnIterator.java | 4 -
.../core/astyanax/MultiTennantColumnFamily.java | 1 -
.../persistence/core/astyanax/ScopedRowKey.java | 1 -
.../astyanax/StringRowCompositeSerializer.java | 35 +-
.../core/hystrix/HystrixCassandra.java | 3 -
.../core/migration/data/DataMigration.java | 31 +-
.../migration/data/DataMigrationException.java | 30 +-
.../migration/data/DataMigrationManager.java | 30 +-
.../data/DataMigrationManagerImpl.java | 3 -
.../data/MigrationInfoSerialization.java | 31 +-
.../data/MigrationInfoSerializationImpl.java | 31 +-
.../migration/schema/MigrationManagerFig.java | 18 +
.../migration/schema/MigrationManagerImpl.java | 18 +
.../core/shard/ExpandingShardLocator.java | 23 +-
.../persistence/core/shard/StringHashUtils.java | 22 +-
.../usergrid/persistence/core/task/Task.java | 19 +-
.../persistence/core/task/TaskExecutor.java | 18 +
.../persistence/model/field/LocationField.java | 8 +
stack/loadtests/README.md | 19 +
stack/loadtests/pom.xml | 4 +-
.../usergrid/scenarios/DeviceScenarios.scala | 2 +-
.../usergrid/scenarios/UserScenarios.scala | 54 ++-
.../org/apache/usergrid/settings/Settings.scala | 2 +-
.../simulations/PostUsersSimulation.scala | 49 +--
.../usergrid/simulations/SetupSimulation.scala | 4 +-
stack/loadtests/src/test/scala/Engine.scala | 4 +-
stack/pom.xml | 1 +
42 files changed, 865 insertions(+), 342 deletions(-)
----------------------------------------------------------------------
[15/50] incubator-usergrid git commit: Finished
EntityVersionCreatedTask and added tests to prove that the issue is fixed.
Posted by sf...@apache.org.
Finished EntityVersionCreatedTask and added tests to prove that the issue is fixed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2c3875c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2c3875c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2c3875c8
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 2c3875c886b550b8c4d8f786877326411272aeb2
Parents: 4a36aa1
Author: grey <gr...@apigee.com>
Authored: Mon Nov 3 09:42:22 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Mon Nov 3 09:42:22 2014 -0800
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../usergrid/corepersistence/GuiceModule.java | 11 ++++
.../events/EntityVersionCreatedHandler.java | 31 +++++++++-
.../corepersistence/StaleIndexCleanupTest.java | 41 +++++++++---
.../corepersistence/TestGuiceModule.java | 5 ++
.../impl/EntityCollectionManagerImpl.java | 2 +
.../impl/EntityVersionCreatedTask.java | 4 +-
.../impl/EntityVersionCreatedTaskTest.java | 65 ++++++++++++--------
.../index/impl/EsEntityIndexBatchImpl.java | 30 ++++++---
.../persistence/index/impl/IndexingUtils.java | 2 +
10 files changed, 142 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 83729f1..3c8b0cb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2640,7 +2640,7 @@ public class CpEntityManager implements EntityManager {
logger.debug("About to Write {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
- cpEntity = ecm .write( cpEntity ).toBlocking().last();
+ cpEntity = ecm.write( cpEntity ).toBlocking().last();
logger.debug("Wrote {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index 21eba41..e723a35 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@ -17,9 +17,13 @@ package org.apache.usergrid.corepersistence;
import com.google.inject.AbstractModule;
import com.google.inject.multibindings.Multibinder;
+import com.google.inject.name.Names;
+
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
+import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
+import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
@@ -55,6 +59,13 @@ public class GuiceModule extends AbstractModule {
Multibinder<EntityVersionDeleted> versionBinder
= Multibinder.newSetBinder(binder(), EntityVersionDeleted.class);
versionBinder.addBinding().to(EntityVersionDeletedHandler.class);
+
+ Multibinder<EntityVersionCreated> versionCreatedMultibinder
+ = Multibinder.newSetBinder( binder(),EntityVersionCreated.class );
+ versionCreatedMultibinder.addBinding().to(EntityVersionCreatedHandler.class);
+
+ bind( Boolean.class ).annotatedWith( Names.named( "defaultConfig" ) ).toInstance( false );
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 64d1125..6f3c507 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -17,8 +17,14 @@
*/
package org.apache.usergrid.corepersistence.events;
+import java.util.Properties;
+
+import org.elasticsearch.common.inject.Guice;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
@@ -28,13 +34,20 @@ import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.model.entity.Entity;
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+
/**
* Clean up stale entity indexes when new version of Entity created. Called when an Entity is
* updated by the Collections module and we react by calling the Query Index module and removing
* any indexes that exist for previous versions of the the Entity.
*/
-public class EntityVersionCreatedHandler implements EntityVersionCreated {
+public class EntityVersionCreatedHandler implements EntityVersionCreated {
+
+ @Inject
+ @Named( "defaultConfig" )
+ private Boolean defaultConfig;
private static final Logger logger = LoggerFactory.getLogger(EntityVersionCreatedHandler.class );
@@ -56,7 +69,19 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
EntityIndexBatch batch = ei.createBatch();
- batch.deindexPreviousVersions( entity );
- batch.execute();
+ //loads test properties, if not found then remove stale entities
+// String[] locations = { "usergrid-properties-context.xml" };
+// ConfigurableApplicationContext appContext =
+// new ClassPathXmlApplicationContext( locations );
+//
+// Properties properties = (Properties)appContext.getBean("properties");
+// String staleString = properties.getProperty("allow.stale.entities","false");
+
+ //if(!staleString.contains( "true" ))
+
+ if(System.getProperty( "allow.stale.entities","false" ).equals( "false" )) {
+ batch.deindexPreviousVersions( entity );
+ batch.execute();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index ff740af..1e64502 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -18,13 +18,20 @@
package org.apache.usergrid.corepersistence;
import com.fasterxml.uuid.UUIDComparator;
+import com.google.inject.Guice;
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.Properties;
import java.util.UUID;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.CoreITSuite;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.count.SimpleBatcher;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
@@ -43,14 +50,21 @@ import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+
import org.junit.Assert;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+
+import org.junit.Before;
+import org.junit.After;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.support.ClassPathXmlApplicationContext;
+//need to create system properties in test that can get applied
/**
* Test on read style clean-up of stale ElasticSearch indexes.
*/
@@ -61,6 +75,18 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
private static final long writeDelayMs = 50;
private static final long readDelayMs = 50;
+ @Before
+ public void before() {
+ //set the batcher to block the submit so we wait for results when testing
+ System.setProperty( "allow.stale.entities","true" );
+
+ }
+
+ @After
+ public void after() {
+ System.clearProperty( "allow.stale.entities" );
+
+ }
/**
* Test that updating an entity causes the entity's version number to change.
@@ -287,7 +313,9 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
@Test
public void testCleanupOnUpdate() throws Exception {
- logger.info("Started testCleanupOnUpdate()");
+ System.setProperty( "allow.stale.entities","false" );
+
+ logger.info( "Started testCleanupOnUpdate()" );
// TODO: turn off index cleanup on read
@@ -337,16 +365,9 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
// query Core Persistence directly for total number of result candidates
crs = queryCollectionCp("things", "select *");
- Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
+ Assert.assertEquals( "Expect candidates without earlier stale entities", numEntities, crs.size() );
- // wait for indexes to be cleared for the deleted entities
- count = 0;
- do {
- Thread.sleep(100);
- crs = queryCollectionCp("things", "select *");
- } while ( crs.size() > 0 || count++ < 14 );
-
- Assert.assertEquals( "Expect no candidates", 0, crs.size() );
+ System.clearProperty( "allow.stale.entities" );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
index 6fce1dd..cdf4c01 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
@@ -19,12 +19,17 @@
package org.apache.usergrid.corepersistence;
import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import com.google.inject.name.Names;
public class TestGuiceModule extends AbstractModule {
@Override
protected void configure() {
+
install( new GuiceModule() );
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 26ab6aa..323b838 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -305,6 +305,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
//we an update, signal the fix
+ taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope,entity));
+ //taskExecutor.submit(entityVersionCleanupFactory.getTask(collectionScope, entityId,entity.getVersion()));
//TODO T.N Change this to fire a task
// Observable.from( new CollectionIoEvent<Id>(collectionScope,
// entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
index 5846c89..7d3beb1 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
@@ -36,9 +36,9 @@ import rx.schedulers.Schedulers;
* Fires events so that all EntityVersionCreated handlers area called.
*/
public class EntityVersionCreatedTask implements Task<Void> {
- private static final Logger logger = LoggerFactory.getLogger( EntityVersionCleanupTask.class );
+ private static final Logger logger = LoggerFactory.getLogger( EntityVersionCreatedTask.class );
- private final Set<EntityVersionCreated> listeners;
+ private Set<EntityVersionCreated> listeners;
private final CollectionScope collectionScope;
private final Entity entity;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 3526fe9..3326856 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -51,6 +51,45 @@ public class EntityVersionCreatedTaskTest {
@Test(timeout=10000)
+ public void noListener()
+ throws ExecutionException, InterruptedException, ConnectionException {
+
+ // create a latch for the event listener, and add it to the list of events
+
+ final int sizeToReturn = 0;
+
+ final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+
+ final EntityVersionCreatedTest eventListener = new EntityVersionCreatedTest(latch);
+
+ final Set<EntityVersionCreated> listeners = mock( Set.class );
+ final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
+
+ when ( listeners.size()).thenReturn( 0 );
+
+ final Id applicationId = new SimpleId( "application" );
+
+ final CollectionScope appScope = new CollectionScopeImpl(
+ applicationId, applicationId, "users" );
+
+ final Id entityId = new SimpleId( "user" );
+ final Entity entity = new Entity( entityId );
+
+ // start the task
+
+ EntityVersionCreatedTask entityVersionCreatedTask =
+ new EntityVersionCreatedTask( appScope, listeners, entity);
+
+ ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
+
+ // wait for the task
+ future.get();
+
+ //mocked listener makes sure that the task is called
+ verify( listeners ).size();
+
+ }
+ @Test(timeout=10000)
public void oneListener()
throws ExecutionException, InterruptedException, ConnectionException {
@@ -106,30 +145,4 @@ public class EntityVersionCreatedTaskTest {
invocationLatch.countDown();
}
}
-
-
-// private static class SlowListener extends EntityVersionCreatedTest {
-// final Semaphore blockLatch;
-//
-// private SlowListener( final CountDownLatch invocationLatch, final Semaphore blockLatch ) {
-// super( invocationLatch );
-// this.blockLatch = blockLatch;
-// }
-//
-//
-// @Override
-// public void versionDeleted( final CollectionScope scope, final Id entityId,
-// final List<MvccEntity> entityVersion ) {
-//
-// //wait for unblock to happen before counting down invocation latches
-// try {
-// blockLatch.acquire();
-// }
-// catch ( InterruptedException e ) {
-// throw new RuntimeException( e );
-// }
-// super.versionDeleted( scope, entityId, entityVersion );
-// }
-// }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index aedee2b..e8ee96f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -23,12 +23,16 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.UUID;
+import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.get.GetRequest;
+import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.FilteredQueryBuilder;
@@ -65,6 +69,7 @@ import com.google.common.base.Joiner;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYVERSION_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
@@ -135,6 +140,9 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
// need prefix here becuase we index UUIDs as strings
entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME,
entity.getId().getUuid().toString().toLowerCase() );
+ //index entity as a long ENTITYVERSION_FIELDNAME
+ //puts it in the index
+ entityAsMap.put( ENTITYVERSION_FIELDNAME, entity.getVersion().timestamp());
// let caller add these fields if needed
// entityAsMap.put("created", entity.getId().getUuid().timestamp();
@@ -217,18 +225,22 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public EntityIndexBatch deindexPreviousVersions(Entity entity){
- FilteredQueryBuilder fqb = QueryBuilders.filteredQuery
- (QueryBuilders.termQuery
- ( STRING_PREFIX + ENTITYID_FIELDNAME,entity.getId().getUuid().toString().toLowerCase() ),
- FilterBuilders.rangeFilter("version").lt( entity.getId().getUuid().timestamp() ));
+ //String helper = System.getProperty( "allow.stale.entities","false" );
+ //"false" gets returned if there is no allow.stale.entities process
+ FilteredQueryBuilder fqb = QueryBuilders.filteredQuery( QueryBuilders
+ .termQuery( STRING_PREFIX + ENTITYID_FIELDNAME,
+ entity.getId().getUuid().toString().toLowerCase() ),
+ FilterBuilders.rangeFilter( ENTITYVERSION_FIELDNAME ).lt( entity.getVersion().timestamp() ) );
- DeleteByQueryResponse response = client.prepareDeleteByQuery( indexName )
- .setQuery( fqb ).execute().actionGet();
+ DeleteByQueryResponse response =
+ client.prepareDeleteByQuery( indexName ).setQuery( fqb ).execute().actionGet();
- logger.debug("Deleted entity {}:{} from all index scopes with response status = {}",
- new Object[] { entity.getId().getType(), entity.getId().getUuid(), response.status().toString() });
+ //error message needs to be retooled so that it describes the entity more througly
+ logger.debug( "Deleted entity {}:{} from all index scopes with response status = {}",
+ new Object[] { entity.getId().getType(), entity.getId().getUuid(), response.status().toString() } );
- maybeFlush();
+ maybeFlush();
+ // }
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c3875c8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index f3e1ba2..8ebcbf3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -39,6 +39,8 @@ public class IndexingUtils {
public static final String ENTITYID_FIELDNAME = "zzz_entityid_zzz";
+ public static final String ENTITYVERSION_FIELDNAME = "zzz_entityversion_zzz";
+
public static final String DOC_ID_SEPARATOR = "|";
public static final String DOC_ID_SEPARATOR_SPLITTER = "\\|";
[33/50] incubator-usergrid git commit: Fix merge issue.
Posted by sf...@apache.org.
Fix merge issue.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c69f0ec3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c69f0ec3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c69f0ec3
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: c69f0ec33ab4683ded8738f95ff50ffd6c521451
Parents: ee260cc
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Nov 24 09:32:15 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Nov 24 09:32:15 2014 -0500
----------------------------------------------------------------------
.../persistence/index/impl/EsEntityIndexBatchImpl.java | 10 +---------
.../usergrid/persistence/index/impl/IndexingUtils.java | 5 +----
2 files changed, 2 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c69f0ec3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 6e68791..b099678 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -29,14 +29,7 @@ import java.util.UUID;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryRequest;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryRequestBuilder;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.client.Client;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.FilteredQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -66,9 +59,8 @@ import org.apache.usergrid.persistence.model.field.value.EntityObject;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_ID_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c69f0ec3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 4319ce4..30b05e4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -47,10 +47,7 @@ public class IndexingUtils {
public static final String ENTITY_CONTEXT_FIELDNAME = "ug_context";
- public static final String ENTITY_ID_FIELDNAME = "ug_entityId";
-
- public static final String ENTITY_VERSION_FIELDNAME = "ug_entityVersion";
-
+ public static final String ENTITYID_ID_FIELDNAME = "ug_entityId";
/**
[11/50] incubator-usergrid git commit: Cleanup EntityVersionCreated
to remove unnecessary fields, add missing ASL header,
and beginning of StaleIndexCleanupTest.testCleanupOnUpdate() method (which is
currently failing).
Posted by sf...@apache.org.
Cleanup EntityVersionCreated to remove unnecessary fields, add missing ASL header, and beginning of StaleIndexCleanupTest.testCleanupOnUpdate() method (which is currently failing).
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7dec37b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7dec37b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7dec37b6
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 7dec37b61df069752ae82efe808978997c40186c
Parents: ac03027
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Oct 30 10:48:44 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Oct 30 10:48:44 2014 -0400
----------------------------------------------------------------------
.../corepersistence/StaleIndexCleanupTest.java | 74 +++++++-
.../collection/event/EntityVersionCreated.java | 14 +-
.../collection/event/EntityVersionDeleted.java | 9 +-
.../impl/EntityVersionCreatedTask.java | 120 +++++-------
.../impl/EntityVersionCreatedTaskTest.java | 187 +++----------------
5 files changed, 153 insertions(+), 251 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7dec37b6/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index d828733..ff740af 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -103,7 +103,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
@Test
public void testCleanupOnRead() throws Exception {
- logger.info("Started testStaleIndexCleanup()");
+ logger.info("Started testCleanupOnRead()");
// TODO: turn off post processing stuff that cleans up stale entities
@@ -274,7 +274,77 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
do {
Thread.sleep(100);
crs = queryCollectionCp("things", "select *");
- } while ( crs.size() > 0 || count++ > 14 );
+ } while ( crs.size() > 0 || count++ < 14 );
+
+ Assert.assertEquals( "Expect no candidates", 0, crs.size() );
+ }
+
+
+ /**
+ * Test that the EntityDeleteImpl cleans up stale indexes on update. Ensures that when an
+ * entity is updated its old indexes are cleared from ElasticSearch.
+ */
+ @Test
+ public void testCleanupOnUpdate() throws Exception {
+
+ logger.info("Started testCleanupOnUpdate()");
+
+ // TODO: turn off index cleanup on read
+
+ final EntityManager em = app.getEntityManager();
+
+ final int numEntities = 10;
+ final int numUpdates = 3;
+
+ // create lots of entities
+ final List<Entity> things = new ArrayList<Entity>(numEntities);
+ for ( int i=0; i<numEntities; i++) {
+ final String thingName = "thing" + i;
+ things.add( em.create("thing", new HashMap<String, Object>() {{
+ put("name", thingName);
+ }}));
+ Thread.sleep( writeDelayMs );
+ }
+ em.refreshIndex();
+
+ CandidateResults crs = queryCollectionCp( "things", "select *");
+ Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
+
+ // update each one a bunch of times
+ int count = 0;
+
+ List<Entity> maxVersions = new ArrayList<>(numEntities);
+
+ for ( Entity thing : things ) {
+ Entity toUpdate = null;
+
+ for ( int j=0; j<numUpdates; j++) {
+ toUpdate = em.get( thing.getUuid() );
+ toUpdate.setProperty( "property" + j, RandomStringUtils.randomAlphanumeric(10));
+
+ em.update(toUpdate);
+
+ Thread.sleep( writeDelayMs );
+ count++;
+ if ( count % 100 == 0 ) {
+ logger.info("Updated {} of {} times", count, numEntities * numUpdates);
+ }
+ }
+
+ maxVersions.add( toUpdate );
+ }
+ em.refreshIndex();
+
+ // query Core Persistence directly for total number of result candidates
+ crs = queryCollectionCp("things", "select *");
+ Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
+
+ // wait for indexes to be cleared for the deleted entities
+ count = 0;
+ do {
+ Thread.sleep(100);
+ crs = queryCollectionCp("things", "select *");
+ } while ( crs.size() > 0 || count++ < 14 );
Assert.assertEquals( "Expect no candidates", 0, crs.size() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7dec37b6/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionCreated.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionCreated.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionCreated.java
index 2b9c023..4412b0c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionCreated.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionCreated.java
@@ -17,26 +17,22 @@
*/
package org.apache.usergrid.persistence.collection.event;
-
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.model.entity.Entity;
/**
- *
- * Invoked after a new version of an entity has been created. The entity should be a complete
- * view of the entity.
- *
+ * Invoked after a new version of an entity has been created.
+ * The entity should be a complete view of the entity.
*/
public interface EntityVersionCreated {
-
/**
- * The new version of the entity. Note that this should be a fully merged view of the entity.
- * In the case of partial updates, the passed entity should be fully merged with it's previous entries
+ * The new version of the entity. Note that this should be a fully merged view of the entity.
+ * In the case of partial updates, the passed entity should be fully merged with it's previous
+ * entries.
* @param scope The scope of the entity
* @param entity The fully loaded and merged entity
*/
public void versionCreated( final CollectionScope scope, final Entity entity );
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7dec37b6/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
index c4b4408..65a34e0 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
@@ -19,8 +19,6 @@ package org.apache.usergrid.persistence.collection.event;
import java.util.List;
-import java.util.UUID;
-
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -28,8 +26,8 @@ import org.apache.usergrid.persistence.model.entity.Id;
/**
*
- * Invoked when an entity version is removed. Note that this is not a deletion of the entity itself,
- * only the version itself.
+ * Invoked when an entity version is removed. Note that this is not a deletion of the entity
+ * itself, only the version itself.
*
*/
public interface EntityVersionDeleted {
@@ -42,6 +40,7 @@ public interface EntityVersionDeleted {
* @param entityId The entity Id that was removed
* @param entityVersions The versions that are to be removed
*/
- public void versionDeleted(final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions);
+ public void versionDeleted(final CollectionScope scope, final Id entityId,
+ final List<MvccEntity> entityVersions);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7dec37b6/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
index 256e65f..5846c89 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
@@ -1,38 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
package org.apache.usergrid.persistence.collection.impl;
-
-import java.util.Iterator;
-import java.util.List;
import java.util.Set;
-import java.util.UUID;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
-import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
-import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.field.Field;
-
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
import rx.Observable;
import rx.functions.Action1;
import rx.functions.Func1;
@@ -40,63 +33,39 @@ import rx.schedulers.Schedulers;
/**
- * Created by ApigeeCorporation on 10/24/14.
+ * Fires events so that all EntityVersionCreated handlers area called.
*/
public class EntityVersionCreatedTask implements Task<Void> {
- private static final Logger LOG = LoggerFactory.getLogger( EntityVersionCleanupTask.class );
-
+ private static final Logger logger = LoggerFactory.getLogger( EntityVersionCleanupTask.class );
private final Set<EntityVersionCreated> listeners;
-
- private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
- private final MvccEntitySerializationStrategy entitySerializationStrategy;
- private UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
- private final Keyspace keyspace;
-
- private final SerializationFig serializationFig;
-
private final CollectionScope collectionScope;
private final Entity entity;
- private EntityVersionCreatedFactory entityVersionCreatedFactory;
-
-
@Inject
- public EntityVersionCreatedTask( EntityVersionCreatedFactory entityVersionCreatedFactory,
- final SerializationFig serializationFig,
- final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final Keyspace keyspace,
- @Assisted final CollectionScope collectionScope,
+ public EntityVersionCreatedTask( @Assisted final CollectionScope collectionScope,
final Set<EntityVersionCreated> listeners,
@Assisted final Entity entity ) {
- this.entityVersionCreatedFactory = entityVersionCreatedFactory;
- this.serializationFig = serializationFig;
- this.logEntrySerializationStrategy = logEntrySerializationStrategy;
- this.entitySerializationStrategy = entitySerializationStrategy;
- this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
- this.keyspace = keyspace;
this.listeners = listeners;
this.collectionScope = collectionScope;
this.entity = entity;
-
}
@Override
public void exceptionThrown( final Throwable throwable ) {
- LOG.error( "Unable to run update task for collection {} with entity {} and version {}",
+ logger.error( "Unable to run update task for collection {} with entity {} and version {}",
new Object[] { collectionScope, entity}, throwable );
}
@Override
public Void rejected() {
- //Our task was rejected meaning our queue was full. We need this operation to run,
- // so we'll run it in our current thread
+
+ // Our task was rejected meaning our queue was full.
+ // We need this operation to run, so we'll run it in our current thread
try {
call();
}
@@ -107,6 +76,7 @@ public class EntityVersionCreatedTask implements Task<Void> {
return null;
}
+
@Override
public Void call() throws Exception {
@@ -114,7 +84,9 @@ public class EntityVersionCreatedTask implements Task<Void> {
return null;
}
+
private void fireEvents() {
+
final int listenerSize = listeners.size();
if ( listenerSize == 0 ) {
@@ -126,25 +98,25 @@ public class EntityVersionCreatedTask implements Task<Void> {
return;
}
- LOG.debug( "Started firing {} listeners", listenerSize );
+ logger.debug( "Started firing {} listeners", listenerSize );
+
//if we have more than 1, run them on the rx scheduler for a max of 8 operations at a time
- Observable.from(listeners)
- .parallel( new Func1<Observable<EntityVersionCreated
- >, Observable<EntityVersionCreated>>() {
-
- @Override
- public Observable<EntityVersionCreated> call(
- final Observable<EntityVersionCreated> entityVersionCreatedObservable ) {
-
- return entityVersionCreatedObservable.doOnNext( new Action1<EntityVersionCreated>() {
- @Override
- public void call( final EntityVersionCreated listener ) {
- listener.versionCreated(collectionScope,entity);
- }
- } );
- }
- }, Schedulers.io() ).toBlocking().last();
-
- LOG.debug( "Finished firing {} listeners", listenerSize );
+ Observable.from(listeners).parallel(
+ new Func1<Observable<EntityVersionCreated>, Observable<EntityVersionCreated>>() {
+
+ @Override
+ public Observable<EntityVersionCreated> call(
+ final Observable<EntityVersionCreated> entityVersionCreatedObservable ) {
+
+ return entityVersionCreatedObservable.doOnNext( new Action1<EntityVersionCreated>() {
+ @Override
+ public void call( final EntityVersionCreated listener ) {
+ listener.versionCreated(collectionScope,entity);
+ }
+ } );
+ }
+ }, Schedulers.io() ).toBlocking().last();
+
+ logger.debug( "Finished firing {} listeners", listenerSize );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7dec37b6/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 21895ba..3526fe9 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -1,45 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
package org.apache.usergrid.persistence.collection.impl;
-
-import java.util.ArrayList;
-import java.util.HashSet;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import java.util.Iterator;
-import java.util.List;
import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Semaphore;
-
-import org.junit.AfterClass;
-import org.junit.Test;
-
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
-import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
-import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.util.LogEntryMock;
import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.google.common.base.Optional;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.same;
+import org.junit.AfterClass;
+import org.junit.Test;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -57,125 +49,22 @@ public class EntityVersionCreatedTaskTest {
taskExecutor.shutdown();
}
- @Test(timeout=10000)
- public void noListener()//why does it matter if it has a version or not without a listener
- throws ExecutionException, InterruptedException, ConnectionException {
-
-
- final SerializationFig serializationFig = mock( SerializationFig.class );
-
- when( serializationFig.getBufferSize() ).thenReturn( 10 );
-
- final MvccEntitySerializationStrategy ess =
- mock( MvccEntitySerializationStrategy.class );
-
- final MvccLogEntrySerializationStrategy less =
- mock( MvccLogEntrySerializationStrategy.class );
-
- final Keyspace keyspace = mock( Keyspace.class );
-
- final MutationBatch entityBatch = mock( MutationBatch.class );
-
- final MutationBatch logBatch = mock( MutationBatch.class );
-
- final EntityVersionCreatedFactory entityVersionCreatedFactory = mock(EntityVersionCreatedFactory.class);
-
- when( keyspace.prepareMutationBatch() )
- .thenReturn( mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch )
- .thenReturn( logBatch );
-
- // intentionally no events
- final Set<EntityVersionCreated> listeners = mock( Set.class );//new HashSet<EntityVersionCreated>();
-
- when ( listeners.size()).thenReturn( 0 );
-
- final Id applicationId = new SimpleId( "application" );
-
- final CollectionScope appScope = new CollectionScopeImpl(
- applicationId, applicationId, "users" );
-
- final Id entityId = new SimpleId( "user" );
-
- final Entity entity = new Entity( entityId );
-
- final MvccEntity mvccEntity = new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
- MvccEntity.Status.COMPLETE,entity );
-
-
- // mock up a single log entry for our first test
- final LogEntryMock logEntryMock =
- LogEntryMock.createLogEntryMock(less, appScope, entityId, 2 );
-
- final UUID version = logEntryMock.getEntries().iterator().next().getVersion();
-
- final UniqueValueSerializationStrategy uvss =
- mock( UniqueValueSerializationStrategy.class );
-
- EntityVersionCreatedTask entityVersionCreatedTask =
- new EntityVersionCreatedTask(entityVersionCreatedFactory,
- serializationFig,
- less,
- ess,
- uvss,
- keyspace,
- appScope,
- listeners,
- entity);
-
- // start the task
- ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
-
- // wait for the task
- future.get();
-
- //mocked listener makes sure that the task is called
- verify( listeners ).size();
-
- }
@Test(timeout=10000)
public void oneListener()
throws ExecutionException, InterruptedException, ConnectionException {
+ // create a latch for the event listener, and add it to the list of events
- final SerializationFig serializationFig = mock( SerializationFig.class );
-
- when( serializationFig.getBufferSize() ).thenReturn( 10 );
-
- final MvccEntitySerializationStrategy ess =
- mock( MvccEntitySerializationStrategy.class );
-
- final MvccLogEntrySerializationStrategy less =
- mock( MvccLogEntrySerializationStrategy.class );
-
- final Keyspace keyspace = mock( Keyspace.class );
-
- final MutationBatch entityBatch = mock( MutationBatch.class );
-
- final MutationBatch logBatch = mock( MutationBatch.class );
-
- final EntityVersionCreatedFactory entityVersionCreatedFactory = mock(EntityVersionCreatedFactory.class);
-
- when( keyspace.prepareMutationBatch() )
- .thenReturn( mock( MutationBatch.class ) ) // don't care what happens to this one
- .thenReturn( entityBatch )
- .thenReturn( logBatch );
-
- // intentionally no events
-
- //create a latch for the event listener, and add it to the list of events
final int sizeToReturn = 1;
final CountDownLatch latch = new CountDownLatch( sizeToReturn );
final EntityVersionCreatedTest eventListener = new EntityVersionCreatedTest(latch);
- final Set<EntityVersionCreated> listeners = mock( Set.class );//new HashSet<EntityVersionCreated>();
-
+ final Set<EntityVersionCreated> listeners = mock( Set.class );
final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
-
when ( listeners.size()).thenReturn( 1 );
when ( listeners.iterator()).thenReturn( helper );
when ( helper.next() ).thenReturn( eventListener );
@@ -186,37 +75,13 @@ public class EntityVersionCreatedTaskTest {
applicationId, applicationId, "users" );
final Id entityId = new SimpleId( "user" );
-
final Entity entity = new Entity( entityId );
- final MvccEntity mvccEntity = new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
- MvccEntity.Status.COMPLETE,entity );
-
-
- // mock up a single log entry for our first test
- final LogEntryMock logEntryMock =
- LogEntryMock.createLogEntryMock(less, appScope, entityId, 2 );
-
- final UUID version = logEntryMock.getEntries().iterator().next().getVersion();
-
- final UniqueValueSerializationStrategy uvss =
- mock( UniqueValueSerializationStrategy.class );
-
- //when ( listeners.iterator().next().versionCreated(appScope,entity));
-
+ // start the task
- EntityVersionCreatedTask entityVersionCreatedTask =
- new EntityVersionCreatedTask(entityVersionCreatedFactory,
- serializationFig,
- less,
- ess,
- uvss,
- keyspace,
- appScope,
- listeners,
- entity);
+ EntityVersionCreatedTask entityVersionCreatedTask =
+ new EntityVersionCreatedTask( appScope, listeners, entity);
- // start the task
ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
// wait for the task
[30/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/73a3bcca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/73a3bcca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/73a3bcca
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 73a3bcca80923978ad643e331d84a0a26e1e691f
Parents: 717eb15 00eb339
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 18 16:45:50 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 18 16:45:50 2014 -0700
----------------------------------------------------------------------
.../src/test/resources/usergrid-test.properties | 5 +
.../corepersistence/CpEntityManager.java | 11 +-
.../migration/EntityTypeMappingMigrationIT.java | 9 +-
.../migration/GraphShardVersionMigrationIT.java | 12 +-
.../usergrid/persistence/CollectionIT.java | 188 +++++--------------
.../persistence/PerformanceEntityWriteTest.java | 2 +-
.../exception/DataCorruptionException.java | 32 ++++
.../MvccEntitySerializationStrategyImpl.java | 43 ++++-
.../graph/GraphManagerShardConsistencyIT.java | 2 +-
stack/corepersistence/pom.xml | 2 +-
.../impl/EntityConnectionIndexImplTest.java | 2 +
.../persistence/index/impl/EntityIndexTest.java | 1 +
.../src/test/resources/usergrid-UNIT.properties | 6 +
.../IllegalArgumentExceptionMapper.java | 8 +
.../exceptions/ManagementExceptionMapper.java | 13 +-
.../rest/exceptions/ThrowableMapper.java | 12 ++
.../rest/test/resource/OrgUserUUIDWrapper.java | 19 +-
.../rest/test/resource/TestOrganization.java | 16 ++
.../usergrid/cassandra/CassandraResource.java | 2 +
19 files changed, 205 insertions(+), 180 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/73a3bcca/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
[37/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c3261795
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c3261795
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c3261795
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: c3261795af2e0992ae9aecd8332e733dac42e0a8
Parents: 1884cce bd0a1e9
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Nov 25 07:31:05 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Nov 25 07:31:05 2014 -0500
----------------------------------------------------------------------
stack/core/pom.xml | 32 +-
.../corepersistence/CpEntityManager.java | 134 +-
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../corepersistence/CpRelationManager.java | 35 +-
.../usergrid/corepersistence/GuiceModule.java | 1 +
.../migration/EntityDataMigration.java | 147 ++
.../migration/EntityTypeMappingMigration.java | 28 +-
.../migration/GraphShardVersionMigration.java | 104 +-
.../corepersistence/migration/Versions.java | 3 +
.../rx/AllEntitiesInSystemObservable.java | 27 +-
.../rx/ApplicationObservable.java | 65 +-
.../corepersistence/util/CpEntityMapUtils.java | 62 +-
.../corepersistence/util/CpNamingUtils.java | 17 +
.../org/apache/usergrid/CoreITSetupImpl.java | 13 +
.../corepersistence/StaleIndexCleanupTest.java | 6 +-
.../migration/EntityDataMigrationIT.java | 252 ++++
.../migration/EntityTypeMappingMigrationIT.java | 74 +-
.../migration/GraphShardVersionMigrationIT.java | 153 +-
.../rx/AllEntitiesInSystemObservableIT.java | 23 +-
.../usergrid/persistence/LargeEntityIT.java | 112 ++
stack/core/src/test/resources/largeentity.json | 1329 ++++++++++++++++++
.../collection/EntityCollectionManager.java | 2 +-
.../exception/EntityTooLargeException.java | 67 +
.../collection/guice/CollectionModule.java | 2 -
.../impl/EntityCollectionManagerImpl.java | 294 ++--
.../collection/impl/EntityDeletedTask.java | 11 +-
.../impl/EntityVersionCleanupTask.java | 13 +-
.../mvcc/MvccEntitySerializationStrategy.java | 23 +-
.../mvcc/MvccLogEntrySerializationStrategy.java | 3 +-
.../mvcc/stage/delete/MarkCommit.java | 19 +-
.../mvcc/stage/write/WriteCommit.java | 3 +-
.../serialization/SerializationFig.java | 15 +-
.../UniqueValueSerializationStrategy.java | 3 +-
.../serialization/impl/EntityRepairImpl.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 343 ++---
...vccEntitySerializationStrategyProxyImpl.java | 162 +++
.../MvccEntitySerializationStrategyV1Impl.java | 219 +++
.../MvccEntitySerializationStrategyV2Impl.java | 238 ++++
.../MvccLogEntrySerializationStrategyImpl.java | 2 +-
.../serialization/impl/SerializationModule.java | 25 +-
.../serialization/impl/SettingsValidation.java | 58 +
.../UniqueValueSerializationStrategyImpl.java | 2 +-
.../EntityCollectionManagerFactoryTest.java | 2 +-
.../collection/EntityCollectionManagerIT.java | 64 +-
.../EntityCollectionManagerStressTest.java | 2 +-
.../EntityCollectionManagerSyncIT.java | 2 +-
.../collection/guice/MigrationManagerRule.java | 38 -
.../collection/guice/TestCollectionModule.java | 10 +
.../impl/EntityVersionCleanupTaskTest.java | 11 +-
...niqueValueSerializationStrategyImplTest.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +-
.../serialization/EntityRepairImplTest.java | 2 +-
...MvccEntitySerializationStrategyImplTest.java | 87 +-
...cEntitySerializationStrategyProxyV1Test.java | 85 ++
...cEntitySerializationStrategyProxyV2Test.java | 83 ++
...ccEntitySerializationStrategyV1ImplTest.java | 55 +
...ccEntitySerializationStrategyV2ImplTest.java | 55 +
.../MvccEntitySerializationStrategyV2Test.java | 229 +++
.../impl/MvccLESSTransientTest.java | 55 +-
...ccLogEntrySerializationStrategyImplTest.java | 2 +-
.../impl/SettingsValidationTest.java | 131 ++
.../collection/util/EntityHelper.java | 105 ++
.../src/test/resources/log4j.properties | 21 +-
.../persistence/core/astyanax/CassandraFig.java | 12 +
.../persistence/core/astyanax/FieldBuffer.java | 70 +
.../core/astyanax/FieldBufferBuilder.java | 87 ++
.../core/astyanax/FieldBufferParser.java | 81 ++
.../core/astyanax/FieldBufferSerializer.java | 134 ++
.../core/guicyfig/SetConfigTestBypass.java | 108 ++
.../astyanax/FieldBufferSerializerTest.java | 126 ++
.../core/guice/MaxMigrationModule.java | 39 +
.../core/guice/MaxMigrationVersion.java | 40 +
.../core/guice/MigrationManagerRule.java | 19 +-
.../persistence/graph/GraphManagerIT.java | 2 +-
.../persistence/graph/GraphManagerLoadTest.java | 2 +-
.../graph/GraphManagerShardingIT.java | 2 +-
.../graph/GraphManagerStressTest.java | 2 +-
.../usergrid/persistence/graph/SimpleTest.java | 2 +-
.../graph/guice/TestGraphModule.java | 11 +
.../graph/impl/EdgeDeleteListenerTest.java | 2 +-
.../graph/impl/NodeDeleteListenerTest.java | 2 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 2 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 2 +-
.../EdgeMetadataSerializationTest.java | 2 +-
.../EdgeSerializationChopTest.java | 2 +-
.../serialization/EdgeSerializationTest.java | 2 +-
.../serialization/NodeSerializationTest.java | 2 +-
.../impl/shard/EdgeShardSerializationTest.java | 2 +-
.../NodeShardCounterSerializationTest.java | 2 +-
.../persistence/map/MapManagerTest.java | 2 +-
.../persistence/model/field/LocationField.java | 3 -
.../index/impl/EntityIndexMapUtils.java | 4 +-
.../persistence/queue/QueueManagerTest.java | 26 +-
.../collection/groups/GroupResourceIT.java | 182 +--
.../applications/queries/GeoPagingTest.java | 41 +-
.../applications/queries/basicGeoTests.java | 184 +++
97 files changed, 5219 insertions(+), 1150 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5a193e5,b2d854b..c3e5bbe
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -69,7 -69,7 +69,6 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
--import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.entities.Application;
@@@ -214,23 -215,23 +214,22 @@@ public class CpEntityManager implement
// set to false for now
this.skipAggregateCounters = false;
-- int entityCacheSize =
-- Integer.parseInt( cass.getProperties().getProperty( "usergrid.entity_cache_size", "100" ) );
++ int entityCacheSize = Integer.parseInt(
++ cass.getProperties().getProperty( "usergrid.entity_cache_size", "100" ) );
-- int entityCacheTimeout =
-- Integer.parseInt( cass.getProperties().getProperty( "usergrid.entity_cache_timeout_ms", "500" ) );
++ int entityCacheTimeout = Integer.parseInt(
++ cass.getProperties().getProperty( "usergrid.entity_cache_timeout_ms", "500" ) );
this.entityCache = CacheBuilder.newBuilder().maximumSize( entityCacheSize )
-- .expireAfterWrite( entityCacheTimeout, TimeUnit.MILLISECONDS )
-- .build( new CacheLoader<EntityScope, org.apache.usergrid.persistence.model
-- .entity.Entity>() {
-- public org.apache.usergrid.persistence.model.entity.Entity load(
-- EntityScope es ) {
-- return managerCache.getEntityCollectionManager( es.scope )
-- .load( es.entityId ).toBlocking()
-- .lastOrDefault( null );
-- }
-- } );
++ .expireAfterWrite(entityCacheTimeout, TimeUnit.MILLISECONDS)
++ .build(new CacheLoader<EntityScope, org.apache.usergrid.persistence.model.entity.Entity>() {
++ public org.apache.usergrid.persistence.model.entity.Entity load(
++ EntityScope es) {
++ return managerCache.getEntityCollectionManager(es.scope)
++ .load(es.entityId).toBlocking()
++ .lastOrDefault(null);
++ }
++ });
}
@@@ -347,8 -348,8 +346,8 @@@
*/
@Metered( group = "core", name = "EntityManager_create" )
@TraceParticipant
-- public <A extends Entity> A create( String entityType, Class<A> entityClass, Map<String, Object> properties,
-- UUID importId ) throws Exception {
++ public <A extends Entity> A create( String entityType, Class<A> entityClass,
++ Map<String, Object> properties, UUID importId ) throws Exception {
UUID timestampUuid = importId != null ? importId : UUIDUtils.newTimeUUID();
@@@ -372,13 -373,9 +371,10 @@@
}
Id id = new SimpleId( entityRef.getUuid(), entityRef.getType() );
- String collectionName = CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() );
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- collectionName );
- CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), entityRef.getType());
++ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
++ applicationScope.getApplication(), entityRef.getType());
- EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
// if ( !UUIDUtils.isTimeBased( id.getUuid() ) ) {
// throw new IllegalArgumentException(
@@@ -390,10 -387,10 +386,10 @@@
if ( cpEntity == null ) {
if ( logger.isDebugEnabled() ) {
logger.debug( "FAILED to load entity {}:{} from scope\n app {}\n owner {}\n name {}",
-- new Object[] {
-- id.getType(), id.getUuid(), collectionScope.getApplication(),
-- collectionScope.getOwner(), collectionScope.getName()
-- } );
++ new Object[] {
++ id.getType(), id.getUuid(), collectionScope.getApplication(),
++ collectionScope.getOwner(), collectionScope.getName()
++ } );
}
return null;
}
@@@ -457,13 -454,10 +453,11 @@@
String type = Schema.getDefaultSchema().getEntityType( entityClass );
Id id = new SimpleId( entityId, type );
- String collectionName = CpNamingUtils.getCollectionScopeNameFromEntityType( type );
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- collectionName );
- EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
- CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), type);
++ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
++ applicationScope.getApplication(), type);
+
// if ( !UUIDUtils.isTimeBased( id.getUuid() ) ) {
// throw new IllegalArgumentException(
@@@ -522,18 -516,17 +516,22 @@@
public void update( Entity entity ) throws Exception {
// first, update entity index in its own collection scope
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( entity.getType() ) );
+
- CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), entity.getType());
++ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
++ applicationScope.getApplication(), entity.getType());
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
if ( logger.isDebugEnabled() ) {
-- logger.debug( "Updating entity {}:{} from scope\n app {}\n owner {}\n name {}", new Object[] {
-- entityId.getType(), entityId.getUuid(), collectionScope.getApplication(),
-- collectionScope.getOwner(), collectionScope.getName()
-- } );
++ logger.debug( "Updating entity {}:{} from scope\n app {}\n owner {}\n name {}",
++ new Object[] {
++ entityId.getType(),
++ entityId.getUuid(),
++ collectionScope.getApplication(),
++ collectionScope.getOwner(),
++ collectionScope.getName()
++ } );
}
// if ( !UUIDUtils.isTimeBased( entityId.getUuid() ) ) {
@@@ -590,9 -583,7 +588,8 @@@
private Observable deleteAsync( EntityRef entityRef ) throws Exception {
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
- CollectionScope collectionScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), entityRef.getType() );
++ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
++ applicationScope.getApplication(), entityRef.getType() );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
@@@ -866,9 -857,9 +863,8 @@@
}
-- private Iterable<EntityRef> getEntityRefsForUniqueProperty( String collName, String propName, String alias )
-- throws Exception {
--
++ private Iterable<EntityRef> getEntityRefsForUniqueProperty(
++ String collName, String propName, String alias ) throws Exception {
final Id id = getIdForUniqueEntityField( collName, propName, alias );
@@@ -900,15 -891,15 +896,17 @@@
get( entityRef ).getType();
}
catch ( Exception e ) {
-- logger.error( "Unable to load entity " + entityRef.getType() + ":" + entityRef.getUuid(), e );
++ logger.error( "Unable to load entity " + entityRef.getType()
++ + ":" + entityRef.getUuid(), e );
}
if ( entityRef == null ) {
-- throw new EntityNotFoundException( "Entity " + entityId.toString() + " cannot be verified" );
++ throw new EntityNotFoundException(
++ "Entity " + entityId.toString() + " cannot be verified" );
}
if ( ( entityType != null ) && !entityType.equalsIgnoreCase( entityRef.getType() ) ) {
throw new UnexpectedEntityTypeException(
-- "Entity " + entityId + " is not the expected type, expected " + entityType + ", found "
-- + entityRef.getType() );
++ "Entity " + entityId + " is not the expected type, expected "
++ + entityType + ", found " + entityRef.getType() );
}
}
return entityRef;
@@@ -924,7 -915,7 +922,8 @@@
@Override
-- public List<Entity> getPartialEntities( Collection<UUID> ids, Collection<String> properties ) throws Exception {
++ public List<Entity> getPartialEntities(
++ Collection<UUID> ids, Collection<String> properties ) throws Exception {
throw new UnsupportedOperationException( "Not supported yet." );
}
@@@ -938,15 -929,15 +937,16 @@@
@Override
-- public void setProperty( EntityRef entityRef, String propertyName, Object propertyValue ) throws Exception {
++ public void setProperty(
++ EntityRef entityRef, String propertyName, Object propertyValue ) throws Exception {
setProperty( entityRef, propertyName, propertyValue, false );
}
@Override
-- public void setProperty( EntityRef entityRef, String propertyName, Object propertyValue, boolean override )
-- throws Exception {
++ public void setProperty( EntityRef entityRef, String propertyName, Object propertyValue,
++ boolean override ) throws Exception {
if ( ( propertyValue instanceof String ) && ( ( String ) propertyValue ).equals( "" ) ) {
propertyValue = null;
@@@ -954,8 -945,8 +954,8 @@@
Entity entity = get( entityRef );
-- propertyValue =
-- Schema.getDefaultSchema().validateEntityPropertyValue( entity.getType(), propertyName, propertyValue );
++ propertyValue = Schema.getDefaultSchema().validateEntityPropertyValue(
++ entity.getType(), propertyName, propertyValue );
entity.setProperty( propertyName, propertyValue );
entity.setProperty( PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis( UUIDUtils.newTimeUUID() ) );
@@@ -1010,15 -1001,10 +1010,12 @@@
@Override
public void deleteProperty( EntityRef entityRef, String propertyName ) throws Exception {
- CollectionScope collectionScope = getCollectionScopeNameFromEntityType(getApplicationScope().getApplication(), entityRef.getType());
+
- String collectionName = CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() );
-
- CollectionScope collectionScope =
- new CollectionScopeImpl( getApplicationScope().getApplication(), getApplicationScope().getApplication(),
- collectionName );
++ CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
++ getApplicationScope().getApplication(), entityRef.getType());
IndexScope defaultIndexScope = new IndexScopeImpl( getApplicationScope().getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+ getCollectionScopeNameFromEntityType( entityRef.getType() ) );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
EntityIndex ei = managerCache.getEntityIndex( getApplicationScope() );
@@@ -1062,15 -1048,15 +1059,16 @@@
@Override
-- public void addToDictionary( EntityRef entityRef, String dictionaryName, Object elementValue ) throws Exception {
++ public void addToDictionary( EntityRef entityRef, String dictionaryName,
++ Object elementValue ) throws Exception {
addToDictionary( entityRef, dictionaryName, elementValue, null );
}
@Override
-- public void addToDictionary( EntityRef entityRef, String dictionaryName, Object elementName, Object elementValue )
-- throws Exception {
++ public void addToDictionary( EntityRef entityRef, String dictionaryName, Object elementName,
++ Object elementValue ) throws Exception {
if ( elementName == null ) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index a57158f,018a9b7..2ff6c4f
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@@ -18,15 -19,10 +18,16 @@@ package org.apache.usergrid.corepersist
import com.google.inject.AbstractModule;
import com.google.inject.multibindings.Multibinder;
+ import org.apache.usergrid.corepersistence.migration.EntityDataMigration;
import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
import org.apache.usergrid.corepersistence.migration.GraphShardVersionMigration;
+import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
+import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
+import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.collection.event.EntityDeleted;
+import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
+import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index e1c0214,9d0c9e6..f159146
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@@ -51,17 -51,13 +51,18 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.model.entity.SimpleId;
import com.fasterxml.uuid.UUIDComparator;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+import org.junit.After;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.junit.Before;
+//need to create system properties in test that can get applied
/**
* Test on read style clean-up of stale ElasticSearch indexes.
*/
@@@ -382,13 -221,9 +383,10 @@@ public class StaleIndexCleanupTest exte
EntityManager em = app.getEntityManager();
- CollectionScope cs = new CollectionScopeImpl(
- new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
- new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
- CpNamingUtils.getCollectionScopeNameFromEntityType( eref.getType() ) );
+ CollectionScope cs = getCollectionScopeNameFromEntityType( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ), eref.getType() );
- EntityCollectionManagerFactory ecmf = CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class );
+ EntityCollectionManagerFactory ecmf =
+ CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class );
EntityCollectionManager ecm = ecmf.createCollectionManager( cs );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index e47d460,534d7a6..4de18fe
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@@ -43,7 -45,7 +43,7 @@@ public interface EntityCollectionManage
/**
* MarkCommit the entity and remove it's indexes with the given entity id
*/
-- public Observable<Void> delete( Id entityId );
++ public Observable<Id> delete( Id entityId );
/**
* Load the entity with the given entity Id
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index 4c96303,8015ca9..1c3e258
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@@ -18,8 -18,6 +18,7 @@@
package org.apache.usergrid.persistence.collection.guice;
- import java.util.Collections;
+
import org.safehaus.guicyfig.GuicyFigModule;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
@@@ -50,9 -43,7 +49,8 @@@ import com.google.inject.Inject
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.google.inject.assistedinject.FactoryModuleBuilder;
-
+import com.google.inject.multibindings.Multibinder;
- import java.util.List;
+import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 437c0d3,d54c5f7..a89924a
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -60,10 -65,6 +65,12 @@@ import com.netflix.astyanax.connectionp
import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.CqlResult;
import com.netflix.astyanax.serializers.StringSerializer;
++import org.apache.usergrid.persistence.collection.EntityDeletedFactory;
++import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
++import org.apache.usergrid.persistence.collection.EntityVersionCreatedFactory;
+import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
- import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
++import org.apache.usergrid.persistence.core.task.Task;
+import org.apache.usergrid.persistence.core.task.TaskExecutor;
- import org.apache.usergrid.persistence.core.util.Health;
import rx.Observable;
import rx.Subscriber;
@@@ -109,24 -106,16 +117,26 @@@ public class EntityCollectionManagerImp
@Inject
- public EntityCollectionManagerImpl(
- @Write final WriteStart writeStart,
- @WriteUpdate final WriteStart writeUpdate,
- final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit, final RollbackAction rollback,
- final MarkStart markStart, final MarkCommit markCommit,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace,
- final SerializationFig config,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
- final EntityVersionCreatedFactory entityVersionCreatedFactory,
- final EntityDeletedFactory entityDeletedFactory,
- public EntityCollectionManagerImpl( @Write final WriteStart writeStart, @WriteUpdate final WriteStart writeUpdate,
- final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit, final RollbackAction rollback,
- final MarkStart markStart, final MarkCommit markCommit,
- @ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace, final SerializationFig config,
- @Assisted final CollectionScope collectionScope ) {
++ public EntityCollectionManagerImpl(
++ @Write final WriteStart writeStart,
++ @WriteUpdate final WriteStart writeUpdate,
++ final WriteUniqueVerify writeVerifyUnique,
++ final WriteOptimisticVerify writeOptimisticVerify,
++ final WriteCommit writeCommit,
++ final RollbackAction rollback,
++ final MarkStart markStart,
++ final MarkCommit markCommit,
++ @ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
++ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
++ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
++ final Keyspace keyspace,
++ final SerializationFig config,
++ final EntityVersionCleanupFactory entityVersionCleanupFactory,
++ final EntityVersionCreatedFactory entityVersionCreatedFactory,
++ final EntityDeletedFactory entityDeletedFactory,
+ @CollectionTaskExecutor final TaskExecutor taskExecutor,
- @Assisted final CollectionScope collectionScope
++ @Assisted final CollectionScope collectionScope
+ ) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
@@@ -176,42 -161,25 +187,42 @@@
// observable = Concurrent.concurrent( observable, Schedulers.io(), new WaitZip(),
// writeVerifyUnique, writeOptimisticVerify );
- observable.map(writeCommit).doOnNext(new Action1<Entity>() {
- // return the commit result.
- return observable.map( writeCommit ).doOnError( rollback );
++ return observable.map(writeCommit).doOnNext(new Action1<Entity>() {
+ @Override
+ public void call(final Entity entity) {
+ //TODO fire the created task first then the entityVersioncleanup
+ taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope,entity));
+ taskExecutor.submit(entityVersionCleanupFactory.getTask(collectionScope, entityId,entity.getVersion()));
+ //post-processing to come later. leave it empty for now.
+ }
+ }).doOnError(rollback);
-
-
- // return the commit result.
- return observable.map(writeCommit).doOnError(rollback);
}
@Override
- public Observable<Void> delete(final Id entityId) {
-
- Preconditions.checkNotNull(entityId, "Entity id is required in this stage");
- Preconditions.checkNotNull(entityId.getUuid(), "Entity id is required in this stage");
- Preconditions.checkNotNull(entityId.getType(), "Entity type is required in this stage");
-
- public Observable<Void> delete( final Id entityId ) {
++ public Observable<Id> delete( final Id entityId ) {
+
+ Preconditions.checkNotNull( entityId, "Entity id is required in this stage" );
+ Preconditions.checkNotNull( entityId.getUuid(), "Entity id is required in this stage" );
+ Preconditions.checkNotNull( entityId.getType(), "Entity type is required in this stage" );
+
- return Observable.from( new CollectionIoEvent<Id>( collectionScope, entityId ) ).map( markStart )
- .doOnNext( markCommit ).map( new Func1<CollectionIoEvent<MvccEntity>, Void>() {
- @Override
- public Void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
- return null;
- }
- } );
++ Observable<Id> o = Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId))
++ .map( markStart)
++ .doOnNext( markCommit)
++ .map( new Func1<CollectionIoEvent<MvccEntity>, Id>() {
++
++ @Override
++ public Id call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
++ MvccEntity entity = mvccEntityCollectionIoEvent.getEvent();
++ Task<Void> task = entityDeletedFactory
++ .getTask( collectionScope, entity.getId(), entity.getVersion());
++ taskExecutor.submit(task);
++ return entity.getId();
++ }
++ }
++ );
+
- Observable<Void> o = Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId)).map(markStart)
- .doOnNext(markCommit).map(new Func1<CollectionIoEvent<MvccEntity>, Void>() {
- @Override
- public Void call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
- MvccEntity entity = mvccEntityCollectionIoEvent.getEvent();
- Task<Void> task = entityDeletedFactory.getTask(
- collectionScope,entity.getId(),entity.getVersion());
- taskExecutor.submit(task);
- return null;
- }
- });
+ return o;
}
@@@ -289,23 -261,30 +304,31 @@@
final Id entityId = entity.getId();
- ValidationUtils.verifyIdentity(entityId);
+
+ ValidationUtils.verifyIdentity( entityId );
// create our observable and start the write
- CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>(collectionScope, entity);
+ CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( collectionScope, entity );
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner(writeData, writeUpdate);
- return observable.map(writeCommit).doOnNext(new Action1<Entity>() {
+ Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeUpdate );
+
+
+ return observable.map( writeCommit ).doOnNext( new Action1<Entity>() {
@Override
- public void call(final Entity entity) {
- logger.debug("sending entity to the queue");
+ public void call( final Entity entity ) {
+ logger.debug( "sending entity to the queue" );
//we an update, signal the fix
+ taskExecutor.submit(entityVersionCreatedFactory.getTask(collectionScope,entity));
+ //TODO T.N Change this to fire a task
+ // Observable.from( new CollectionIoEvent<Id>(collectionScope,
+ // entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
+
+
}
- }).doOnError(rollback);
+ } ).doOnError( rollback );
}
@@@ -381,5 -358,4 +402,4 @@@
return Health.RED;
}
-
--}
++}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
index 284cf5b,0000000..9ff4f56
mode 100644,000000..100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityDeletedTask.java
@@@ -1,147 -1,0 +1,148 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.collection.impl;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import com.netflix.astyanax.MutationBatch;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
+import org.apache.usergrid.persistence.collection.event.EntityDeleted;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.core.task.Task;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Func1;
+import rx.schedulers.Schedulers;
+
+import java.util.Set;
+import java.util.UUID;
++import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+
+
+/**
+ * Fires Cleanup Task
+ */
+public class EntityDeletedTask implements Task<Void> {
+ private static final Logger LOG = LoggerFactory.getLogger(EntityDeletedTask.class);
+
- private EntityVersionCleanupFactory entityVersionCleanupFactory;
- private MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
- private MvccEntitySerializationStrategy entitySerializationStrategy;
- private Set<EntityDeleted> listeners;
++ private final EntityVersionCleanupFactory entityVersionCleanupFactory;
++ private final MvccLogEntrySerializationStrategy logEntrySerializationStrategy;
++ private final MvccEntitySerializationStrategy entitySerializationStrategy;
++ private final Set<EntityDeleted> listeners;
+ private final CollectionScope collectionScope;
+ private final Id entityId;
+ private final UUID version;
+
+
+ @Inject
+ public EntityDeletedTask(
+ EntityVersionCleanupFactory entityVersionCleanupFactory,
+ final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
++ @ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final Set<EntityDeleted> listeners, // MUST be a set or Guice will not inject
+ @Assisted final CollectionScope collectionScope,
+ @Assisted final Id entityId,
+ @Assisted final UUID version) {
+
+ this.entityVersionCleanupFactory = entityVersionCleanupFactory;
+ this.logEntrySerializationStrategy = logEntrySerializationStrategy;
+ this.entitySerializationStrategy = entitySerializationStrategy;
+ this.listeners = listeners;
+ this.collectionScope = collectionScope;
+ this.entityId = entityId;
+ this.version = version;
+ }
+
+
+ @Override
+ public void exceptionThrown(Throwable throwable) {
+ LOG.error( "Unable to run update task for collection {} with entity {} and version {}",
+ new Object[] { collectionScope, entityId, version }, throwable );
+ }
+
+
+ @Override
+ public Void rejected() {
+ try {
+ call();
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Exception thrown in call task", e );
+ }
+
+ return null;
+ }
+
+
+ @Override
+ public Void call() throws Exception {
+
+ entityVersionCleanupFactory.getTask( collectionScope, entityId, version ).call();
+
+ fireEvents();
+ final MutationBatch entityDelete = entitySerializationStrategy.delete(collectionScope, entityId, version);
+ final MutationBatch logDelete = logEntrySerializationStrategy.delete(collectionScope, entityId, version);
+ entityDelete.execute();
+ logDelete.execute();
+
+ return null;
+ }
+
+
+ private void fireEvents() {
+ final int listenerSize = listeners.size();
+
+ if ( listenerSize == 0 ) {
+ return;
+ }
+
+ if ( listenerSize == 1 ) {
+ listeners.iterator().next().deleted( collectionScope, entityId,version );
+ return;
+ }
+
+ LOG.debug( "Started firing {} listeners", listenerSize );
+
+ //if we have more than 1, run them on the rx scheduler for a max of 8 operations at a time
+ Observable.from(listeners)
+ .parallel( new Func1<Observable<EntityDeleted>, Observable<EntityDeleted>>() {
+
+ @Override
+ public Observable<EntityDeleted> call(
+ final Observable<EntityDeleted> entityVersionDeletedObservable ) {
+
+ return entityVersionDeletedObservable.doOnNext( new Action1<EntityDeleted>() {
+ @Override
+ public void call( final EntityDeleted listener ) {
+ listener.deleted(collectionScope, entityId, version);
+ }
+ } );
+ }
+ }, Schedulers.io() ).toBlocking().last();
+
+ LOG.debug( "Finished firing {} listeners", listenerSize );
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index d4bf6e6,422a155..efecdeb
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@@ -45,7 -22,6 +45,8 @@@ import org.apache.usergrid.persistence.
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import java.util.Set;
++import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import rx.Observable;
import rx.functions.Action1;
@@@ -75,17 -51,11 +76,17 @@@ public class EntityVersionCleanupTask i
private final UUID version;
- public EntityVersionCleanupTask( final SerializationFig serializationFig,
- final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final Keyspace keyspace, final List<EntityVersionDeleted> listeners,
- final CollectionScope scope, final Id entityId, final UUID version ) {
+ @Inject
+ public EntityVersionCleanupTask(
+ final SerializationFig serializationFig,
+ final MvccLogEntrySerializationStrategy logEntrySerializationStrategy,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
++ @ProxyImpl final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final Keyspace keyspace,
+ final Set<EntityVersionDeleted> listeners, // MUST be a set or Guice will not inject
+ @Assisted final CollectionScope scope,
+ @Assisted final Id entityId,
+ @Assisted final UUID version ) {
this.serializationFig = serializationFig;
this.logEntrySerializationStrategy = logEntrySerializationStrategy;
@@@ -123,79 -93,66 +124,81 @@@
@Override
public Void call() throws Exception {
-
-
- final UUID maxVersion = version;
-
-
- Observable<MvccLogEntry> versions = Observable.create( new ObservableIterator( "versionIterators" ) {
- @Override
- protected Iterator getIterator() {
- return new LogEntryIterator( logEntrySerializationStrategy, scope, entityId, maxVersion,
- serializationFig.getBufferSize() );
- }
- } );
-
-
- //get the uuid from the version
- versions.map( new Func1<MvccLogEntry, UUID>() {
- @Override
- public UUID call( final MvccLogEntry mvccLogEntry ) {
- return mvccLogEntry.getVersion();
- }
- } )
- //buffer our versions
- .buffer( serializationFig.getBufferSize() )
- //for each buffer set, delete all of them
- .doOnNext( new Action1<List<UUID>>() {
- @Override
- public void call( final List<UUID> versions ) {
-
- //Fire all the listeners
- fireEvents( versions );
-
- MutationBatch entityBatch = keyspace.prepareMutationBatch();
- MutationBatch logBatch = keyspace.prepareMutationBatch();
-
- for ( UUID version : versions ) {
- final MutationBatch entityDelete = entitySerializationStrategy.delete( scope, entityId, version );
-
- entityBatch.mergeShallow( entityDelete );
-
- final MutationBatch logDelete = logEntrySerializationStrategy.delete( scope, entityId, version );
-
- logBatch.mergeShallow( logDelete );
+ //TODO Refactor this logic into a a class that can be invoked from anywhere
+ //load every entity we have history of
+ Observable<List<MvccEntity>> deleteFieldsObservable =
+ Observable.create(new ObservableIterator<MvccEntity>("deleteColumns") {
+ @Override
+ protected Iterator<MvccEntity> getIterator() {
- Iterator<MvccEntity> entities = entitySerializationStrategy
- .load(scope, entityId, version, serializationFig.getBufferSize());
++ Iterator<MvccEntity> entities = entitySerializationStrategy.loadDescendingHistory(
++ scope, entityId, version, 1000); // TODO: what fetchsize should we use here?
+ return entities;
}
- }) //buffer them for efficiency
++ })
++ //buffer them for efficiency
+ .skip(1)
+ .buffer(serializationFig.getBufferSize()).doOnNext(
+ new Action1<List<MvccEntity>>() {
+ @Override
+ public void call(final List<MvccEntity> mvccEntities) {
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+ final MutationBatch entityBatch = keyspace.prepareMutationBatch();
+ final MutationBatch logBatch = keyspace.prepareMutationBatch();
+
+ for (MvccEntity mvccEntity : mvccEntities) {
+ if (!mvccEntity.getEntity().isPresent()) {
+ continue;
+ }
+
+ final UUID entityVersion = mvccEntity.getVersion();
+ final Entity entity = mvccEntity.getEntity().get();
+
+ //remove all unique fields from the index
+ for (final Field field : entity.getFields()) {
+ if (!field.isUnique()) {
+ continue;
+ }
+ final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion);
- final MutationBatch deleteMutation = uniqueValueSerializationStrategy.delete(scope,unique);
++ final MutationBatch deleteMutation =
++ uniqueValueSerializationStrategy.delete(scope,unique);
+ batch.mergeShallow(deleteMutation);
+ }
+
+ final MutationBatch entityDelete = entitySerializationStrategy
+ .delete(scope, entityId, mvccEntity.getVersion());
+ entityBatch.mergeShallow(entityDelete);
+ final MutationBatch logDelete = logEntrySerializationStrategy
+ .delete(scope, entityId, version);
+ logBatch.mergeShallow(logDelete);
+ }
+
+ try {
+ batch.execute();
+ } catch (ConnectionException e1) {
+ throw new RuntimeException("Unable to execute " +
+ "unique value " +
+ "delete", e1);
+ }
+ fireEvents(mvccEntities);
+ try {
+ entityBatch.execute();
+ } catch (ConnectionException e) {
+ throw new RuntimeException("Unable to delete entities in cleanup", e);
+ }
+
+ try {
+ logBatch.execute();
+ } catch (ConnectionException e) {
+ throw new RuntimeException("Unable to delete entities from the log", e);
+ }
-
- try {
- entityBatch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to delete entities in cleanup", e );
- }
-
- try {
- logBatch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to delete entities from the log", e );
}
}
- } ).count().toBlocking().last();
+ );
+
+ final int removedCount = deleteFieldsObservable.count().toBlocking().last();
+
+ logger.debug("Removed unique values for {} entities of entity {}",removedCount,entityId);
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index 994465d,e31aba5..baf2ac3
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@@ -19,8 -19,8 +19,6 @@@
package org.apache.usergrid.persistence.collection.mvcc.stage.delete;
--import java.util.Iterator;
--import java.util.List;
import java.util.UUID;
import org.slf4j.Logger;
@@@ -36,13 -36,14 +34,9 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
--import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
--import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
- import org.apache.usergrid.persistence.core.rx.ObservableIterator;
- import org.apache.usergrid.persistence.model.entity.Entity;
+ import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.rx.ObservableIterator;
-import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
--import org.apache.usergrid.persistence.model.field.Field;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
@@@ -51,7 -52,7 +45,6 @@@ import com.netflix.astyanax.Keyspace
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
--import rx.Observable;
import rx.functions.Action1;
@@@ -119,72 -120,69 +112,70 @@@ public class MarkCommit implements Acti
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to mark entry as deleted" );
}
- //<<<<<<< HEAD
- //=======
-
-
- //TODO Refactor this logic into a a class that can be invoked from anywhere
- //load every entity we have history of
- Observable<List<MvccEntity>> deleteFieldsObservable =
- Observable.create( new ObservableIterator<MvccEntity>( "deleteColumns" ) {
- @Override
- protected Iterator<MvccEntity> getIterator() {
- Iterator<MvccEntity> entities =
- entityStrat.loadDescendingHistory( collectionScope, entityId, entity.getVersion(), 100 );
-
- return entities;
- }
- } ) //buffer them for efficiency
- .buffer( serializationFig.getBufferSize() ).doOnNext(
-
- new Action1<List<MvccEntity>>() {
- @Override
- public void call( final List<MvccEntity> mvccEntities ) {
-
-
- final MutationBatch batch = keyspace.prepareMutationBatch();
-
- for ( MvccEntity mvccEntity : mvccEntities ) {
- if ( !mvccEntity.getEntity().isPresent() ) {
- continue;
- }
-
- final UUID entityVersion = mvccEntity.getVersion();
-
- final Entity entity = mvccEntity.getEntity().get();
-
- //remove all unique fields from the index
- for ( final Field field : entity.getFields() ) {
-
- if(!field.isUnique()){
- continue;
- }
-
- final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
-
- final MutationBatch deleteMutation = uniqueValueStrat.delete(collectionScope, unique );
-
- batch.mergeShallow( deleteMutation );
- }
- }
-
- try {
- batch.execute();
- }
- catch ( ConnectionException e1 ) {
- throw new RuntimeException( "Unable to execute " +
- "unique value " +
- "delete", e1 );
- }
- }
- }
-
-
- );
-
- final int removedCount = deleteFieldsObservable.count().toBlocking().last();
-
- LOG.debug("Removed unique values for {} entities of entity {}", removedCount, entityId );
+ }
+ }
++
+//
+//
+// //TODO Refactor this logic into a a class that can be invoked from anywhere
+// //load every entity we have history of
+// Observable<List<MvccEntity>> deleteFieldsObservable =
+// Observable.create( new ObservableIterator<MvccEntity>( "deleteColumns" ) {
+// @Override
+// protected Iterator<MvccEntity> getIterator() {
+// Iterator<MvccEntity> entities =
+// entityStrat.load( collectionScope, entityId, entity.getVersion(), 100 );
+//
+// return entities;
+// }
+// } ) //buffer them for efficiency
+// .buffer( serializationFig.getBufferSize() ).doOnNext(
+//
+// new Action1<List<MvccEntity>>() {
+// @Override
+// public void call( final List<MvccEntity> mvccEntities ) {
+//
+//
+// final MutationBatch batch = keyspace.prepareMutationBatch();
+//
+// for ( MvccEntity mvccEntity : mvccEntities ) {
+// if ( !mvccEntity.getEntity().isPresent() ) {
+// continue;
+// }
+//
+// final UUID entityVersion = mvccEntity.getVersion();
+//
+// final Entity entity = mvccEntity.getEntity().get();
+//
+// //remove all unique fields from the index
+// for ( final Field field : entity.getFields() ) {
+//
+// if(!field.isUnique()){
+// continue;
+// }
+//
+// final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
+//
+// final MutationBatch deleteMutation = uniqueValueStrat.delete(collectionScope, unique );
+//
+// batch.mergeShallow( deleteMutation );
+// }
+// }
+//
+// try {
+// batch.execute();
+// }
+// catch ( ConnectionException e1 ) {
+// throw new RuntimeException( "Unable to execute " +
+// "unique value " +
+// "delete", e1 );
+// }
+// }
+// }
+//
+//
+// );
+//
+// final int removedCount = deleteFieldsObservable.count().toBlocking().last();
+//
+// LOG.debug("Removed unique values for {} entities of entity {}", removedCount, entityId );
- //>>>>>>> befcdcab6f7f1c83dbcb2a24eddb055c9297d59f
- }
- }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
index 73f1b6d,e2e6876..dd3c013
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/guice/TestCollectionModule.java
@@@ -21,7 -21,12 +21,8 @@@ package org.apache.usergrid.persistence
import org.apache.usergrid.persistence.core.guice.CommonModule;
+ import org.apache.usergrid.persistence.core.guice.MaxMigrationModule;
-import org.apache.usergrid.persistence.core.guice.MaxMigrationVersion;
import org.apache.usergrid.persistence.core.guice.TestModule;
-import org.apache.usergrid.persistence.core.migration.data.DataMigration;
-
-import com.google.inject.multibindings.Multibinder;
public class TestCollectionModule extends TestModule {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c3261795/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
index eae1c54,1fce6e2..d0a87c3
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
@@@ -139,31 -119,22 +139,32 @@@ public class EntityVersionCleanupTaskTe
final MutationBatch newBatch = mock( MutationBatch.class );
- //set up returning a mutator
- when( mvccEntitySerializationStrategy.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
+ // set up returning a mutator
+ when(ess.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
.thenReturn( newBatch );
-
- when( mvccLogEntrySerializationStrategy.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
+ when(less.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
.thenReturn( newBatch );
+ final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- //start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- //wait for the task
- future.get();
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- when( ess.load( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
- //verify it was run
++ when( ess.loadDescendingHistory(
++ same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
+ .thenReturn(mel.iterator() );
+
+ try {
+ cleanupTask.call();
+ }catch(Exception e){
+ Assert.fail( e.getMessage() );
+ }
+
+ // verify it was run
verify( entityBatch ).execute();
verify( logBatch ).execute();
@@@ -239,38 -195,24 +240,38 @@@
//set up returning a mutator
- when( mvccEntitySerializationStrategy.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
+ when(ess.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
.thenReturn( batch );
-
- when( mvccLogEntrySerializationStrategy.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
+ when( mvccLogEntrySerializationStrategy
+ .delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
.thenReturn( batch );
+ final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- //start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- //wait for the task
- future.get();
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- when( ess.load( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
- //verify it was run
- verify( entityBatch, never() ).execute();
++ when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
+ .thenReturn(mel.iterator() );
- verify( logBatch, never() ).execute();
+ //start the task
+ try {
+ cleanupTask.call();
+ }catch(Exception e){
+ Assert.fail( e.getMessage() );
+ }
+
+
+ // These last two verify statements do not make sense. We cannot assert that the entity
+ // and log batches are never called. Even if there are no listeners the entity delete
+ // cleanup task will still run to do the normal cleanup.
+ //
+ // verify( entityBatch, never() ).execute();
+ // verify( logBatch, never() ).execute();
}
@@@ -358,23 -283,11 +359,23 @@@
.thenReturn( batch );
- //start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
+ final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- //wait for the task
- future.get();
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
+
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
+
- when( ess.load( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
++ when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
+ .thenReturn(mel.iterator() );
+
+
+ try {
+ cleanupTask.call();
+ }catch(Exception e){
+ Assert.fail( e.getMessage() );
+ }
//we deleted the version
//verify it was run
@@@ -462,36 -366,27 +463,36 @@@
//set up returning a mutator
- when( mvccEntitySerializationStrategy.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
+ when( ess.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
.thenReturn( batch );
-
- when( mvccLogEntrySerializationStrategy.delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
+ when( mvccLogEntrySerializationStrategy
+ .delete( same( appScope ), same( entityId ), any( UUID.class ) ) )
.thenReturn( batch );
+ final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- //start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
+ Entity entity = new Entity( entityId );
- //wait for the task
- future.get();
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.of(entity)) );
- //we deleted the version
- //verify we deleted everything
- verify( entityBatch, times( sizeToReturn ) ).mergeShallow( any( MutationBatch.class ) );
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.of(entity)) );
- when( ess.load( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
- verify( logBatch, times( sizeToReturn ) ).mergeShallow( any( MutationBatch.class ) );
++ when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
+ .thenReturn(mel.iterator() );
+ try {
+ cleanupTask.call();
+ }catch(Exception e){
+ Assert.fail( e.getMessage() );
+ }
+ //we deleted the version
+ //verify we deleted everything
+ verify( entityBatch, times( 1 ) ).mergeShallow( any( MutationBatch.class ) );
+ verify( logBatch, times( 1 ) ).mergeShallow( any( MutationBatch.class ) );
verify( logBatch ).execute();
@@@ -729,29 -608,41 +730,29 @@@
.thenReturn( batch );
- //start the task
- ListenableFuture<Void> future1 = taskExecutor.submit( firstTask );
-
- //now start another task while the slow running task is running
- ListenableFuture<Void> future2 = taskExecutor.submit( secondTask );
+ final List<MvccEntity> mel = new ArrayList<MvccEntity>();
- //get the second task, we shouldn't have been able to queue it, therefore it should just run in process
- future2.get();
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- /**
- * While we're not done, release latches every 200 ms
- */
- while ( !future1.isDone() ) {
- Thread.sleep( 200 );
- waitSemaphore.release( listenerCount );
- }
+ mel.add( new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(),
+ MvccEntity.Status.DELETED, Optional.fromNullable((Entity)null)) );
- when( ess.load( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
- //wait for the task
- future1.get();
++ when( ess.loadDescendingHistory( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
+ .thenReturn(mel.iterator() );
- //we deleted the version
- //verify we deleted everything
+ try {
+ cleanupTask.rejected();
+ }catch(Exception e){
+ Assert.fail(e.getMessage());
+ }
//we deleted the version
- //verify we deleted everything
- verify( logBatch, times( sizeToReturn* 2 ) ).mergeShallow( any( MutationBatch.class ) );
-
- verify( entityBatch, times( sizeToReturn * 2) ).mergeShallow( any( MutationBatch.class ) );
-
-
- verify( logBatch, times(2) ).execute();
-
- verify( entityBatch, times(2) ).execute();
+ //verify it was run
+ verify( entityBatch ).execute();
+ verify( logBatch ).execute();
//the latch was executed
latch.await();
[29/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Still need to update code to match new methods of custom fields and remove Query objects
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/717eb15d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/717eb15d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/717eb15d
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 717eb15d581c89fe5239c15733bae0588e7da25d
Parents: 1f71cf6 66ab376
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 17 13:11:38 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 17 13:11:38 2014 -0700
----------------------------------------------------------------------
stack/awscluster/src/main/cql/create_locks.cql | 26 -
.../awscluster/src/main/cql/create_usergrid.cql | 28 -
.../main/cql/create_usergrid_applications.cql | 47 -
stack/awscluster/src/main/cql/update_locks.cql | 23 +
.../awscluster/src/main/cql/update_usergrid.cql | 27 +
.../main/cql/update_usergrid_applications.cql | 46 +
.../main/dist/init_instance/create_keyspaces.sh | 61 -
.../main/dist/init_instance/init_db_server.sh | 4 -
.../main/dist/init_instance/init_rest_server.sh | 28 +-
.../main/dist/init_instance/install_yourkit.sh | 6 +-
.../main/dist/init_instance/update_keyspaces.sh | 66 +
.../src/main/groovy/NodeRegistry.groovy | 8 +-
.../src/main/groovy/configure_usergrid.groovy | 2 +-
stack/awscluster/ugcluster-cf.json | 246 +++-
.../main/resources/usergrid-default.properties | 2 +-
.../src/test/resources/usergrid-test.properties | 6 +-
.../corepersistence/CpEntityManager.java | 1292 ++++++++----------
.../corepersistence/CpEntityManagerFactory.java | 134 +-
.../corepersistence/CpManagerCache.java | 132 +-
.../corepersistence/CpRelationManager.java | 542 ++++----
.../usergrid/corepersistence/CpSetup.java | 29 +-
.../usergrid/corepersistence/CpWalker.java | 9 +-
.../usergrid/corepersistence/GuiceModule.java | 2 +
.../HybridEntityManagerFactory.java | 15 +-
.../usergrid/corepersistence/ManagerCache.java | 69 +
.../migration/EntityTypeMappingMigration.java | 103 ++
.../migration/GraphShardVersionMigration.java | 125 +-
.../corepersistence/migration/Versions.java | 9 +-
.../results/FilteringLoader.java | 33 +-
.../results/ResultsLoaderFactory.java | 9 +-
.../results/ResultsLoaderFactoryImpl.java | 13 +-
.../rx/AllEntitiesInSystemObservable.java | 96 ++
.../rx/ApplicationObservable.java | 117 ++
.../rx/EdgesFromSourceObservable.java | 63 +
.../rx/EdgesToTargetObservable.java | 63 +
.../corepersistence/rx/TargetIdObservable.java | 66 +
.../corepersistence/util/CpNamingUtils.java | 78 +-
.../usergrid/persistence/EntityManager.java | 3 +
.../persistence/EntityManagerFactory.java | 12 +
.../usergrid/persistence/SimpleEntityRef.java | 6 +
.../persistence/cassandra/CassandraService.java | 3 +
.../cassandra/EntityManagerFactoryImpl.java | 16 +-
.../cassandra/RelationManagerImpl.java | 18 +-
.../apache/usergrid/ConcurrentCoreITSuite.java | 2 +-
.../usergrid/ConcurrentCoreIteratorITSuite.java | 4 +-
.../org/apache/usergrid/CoreApplication.java | 7 +-
.../org/apache/usergrid/CoreITSetupImpl.java | 1 -
.../java/org/apache/usergrid/CoreITSuite.java | 4 +-
.../batch/job/AbstractSchedulerRuntimeIT.java | 7 +-
.../usergrid/batch/job/SchedulerRuntime1IT.java | 2 +
.../usergrid/batch/job/SchedulerRuntime2IT.java | 2 +
.../usergrid/batch/job/SchedulerRuntime3IT.java | 2 +
.../usergrid/batch/job/SchedulerRuntime4IT.java | 2 +
.../usergrid/batch/job/SchedulerRuntime5IT.java | 2 +
.../usergrid/batch/job/SchedulerRuntime6IT.java | 2 +
.../usergrid/batch/job/SchedulerRuntime7IT.java | 2 +
.../usergrid/batch/job/SchedulerRuntime8IT.java | 2 +-
.../corepersistence/EntityWriteHelper.java | 59 +
.../corepersistence/StaleIndexCleanupTest.java | 182 +--
.../migration/EntityTypeMappingMigrationIT.java | 158 +++
.../migration/GraphShardVersionMigrationIT.java | 212 +++
.../migration/TestProgressObserver.java | 71 +
.../rx/AllEntitiesInSystemObservableIT.java | 140 ++
.../rx/ApplicationObservableTestIT.java | 82 ++
.../rx/EdgesFromSourceObservableIT.java | 140 ++
.../rx/EdgesToTargetObservableIT.java | 150 ++
.../rx/TargetIdObservableTestIT.java | 131 ++
.../usergrid/persistence/CollectionIT.java | 3 +-
.../persistence/EntityConnectionsIT.java | 43 +-
.../org/apache/usergrid/persistence/GeoIT.java | 34 +-
.../PerformanceEntityRebuildIndexTest.java | 6 +-
.../apache/usergrid/persistence/QueryTest.java | 15 +
.../query/AbstractIteratingQueryIT.java | 20 +-
.../query/AllInConnectionNoTypeIT.java | 2 +-
.../resources/usergrid-custom-test.properties | 4 +-
.../migration/data/DataMigrationManager.java | 11 +
.../data/DataMigrationManagerImpl.java | 26 +-
.../data/MigrationInfoSerializationImpl.java | 2 +-
.../core/astyanax/ColumnNameIteratorTest.java | 21 +-
.../MultiKeyColumnNameIteratorTest.java | 20 +-
.../astyanax/MultiRowColumnIteratorTest.java | 20 +-
.../core/cassandra/CassandraRule.java | 114 --
.../data/MigrationInfoSerializationTest.java | 13 +-
.../persistence/core/test/ITRunner.java | 20 +-
.../EdgeMetadataSerializationProxyImpl.java | 2 +-
.../graph/CommittedGraphManagerIT.java | 2 +-
.../persistence/graph/GraphManagerLoadTest.java | 4 +-
.../graph/GraphManagerShardConsistencyIT.java | 6 -
.../graph/GraphManagerShardingIT.java | 4 +-
.../graph/GraphManagerStressTest.java | 4 +-
.../graph/StorageGraphManagerIT.java | 2 +-
.../graph/guice/TestGraphModule.java | 2 +-
.../graph/impl/EdgeDeleteListenerTest.java | 6 +-
.../graph/impl/NodeDeleteListenerTest.java | 6 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 4 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 6 +-
.../EdgeMetaDataSerializationProxyV2Test.java | 7 +-
.../EdgeMetaDataSerializationV1Test.java | 4 +-
.../EdgeMetaDataSerializationV2Test.java | 4 +-
.../EdgeSerializationChopTest.java | 4 +-
.../serialization/EdgeSerializationTest.java | 7 +-
.../serialization/NodeSerializationTest.java | 4 +-
.../PermanentSerializationTest.java | 2 +-
.../impl/shard/EdgeShardSerializationTest.java | 9 +-
.../NodeShardCounterSerializationTest.java | 8 +-
.../map/impl/MapSerializationImpl.java | 4 +-
stack/corepersistence/pom.xml | 2 +-
.../usergrid/persistence/index/EntityIndex.java | 10 +-
.../usergrid/persistence/index/SearchTypes.java | 130 ++
.../index/impl/EsEntityIndexBatchImpl.java | 138 +-
.../index/impl/EsEntityIndexImpl.java | 190 ++-
.../persistence/index/impl/EsQueryVistor.java | 3 +-
.../persistence/index/impl/IndexScopeImpl.java | 14 +-
.../persistence/index/impl/IndexingUtils.java | 107 +-
.../usergrid/persistence/index/query/Query.java | 76 +-
.../index/impl/CorePerformanceIT.java | 40 +-
.../index/impl/ElasticSearchResource.java | 115 +-
.../impl/EntityConnectionIndexImplTest.java | 249 +++-
.../persistence/index/impl/EntityIndexTest.java | 115 +-
.../persistence/index/impl/EsTestUtils.java | 48 +
.../persistence/query/tree/GrammarTreeTest.java | 2 +-
.../src/test/resources/dynamic-test.properties | 2 +-
.../src/test/resources/usergrid-UNIT.properties | 2 +-
.../usergrid/launcher/EmbeddedServerHelper.java | 4 +-
stack/loadtests/README.md | 6 +-
.../datagenerators/EntityDataGenerator.scala | 2 +-
.../datagenerators/FeederGenerator.scala | 56 +-
.../usergrid/scenarios/DeviceScenarios.scala | 34 +-
.../scenarios/NotificationScenarios.scala | 14 +-
.../usergrid/scenarios/UserScenarios.scala | 54 +-
.../org/apache/usergrid/settings/Settings.scala | 17 +-
.../usergrid/simulations/AppSimulation.scala | 8 +-
.../simulations/ConnectionsSimulation.scala | 2 +-
.../simulations/GetEntitySimulation.scala | 2 +-
.../PushNotificationTargetUserSimulation.scala | 2 +-
.../usergrid/simulations/SetupSimulation.scala | 45 +
stack/pom.xml | 7 -
.../apache/usergrid/rest/MigrateResource.java | 165 +++
.../apache/usergrid/rest/SystemResource.java | 63 +-
.../apache/usergrid/rest/AbstractRestIT.java | 22 +-
.../java/org/apache/usergrid/rest/BasicIT.java | 124 +-
.../java/org/apache/usergrid/rest/ITSetup.java | 16 +-
.../org/apache/usergrid/rest/RestITSuite.java | 2 +-
.../apache/usergrid/rest/TestContextSetup.java | 10 +-
.../rest/management/ExportResourceIT.java | 333 +++--
.../rest/management/ManagementResourceIT.java | 143 +-
.../rest/management/OrganizationsIT.java | 327 +++--
.../rest/management/RegistrationIT.java | 61 +-
.../rest/test/resource/OrgUserUUIDWrapper.java | 29 +
.../rest/test/resource/TestContext.java | 21 +-
.../rest/test/resource/TestOrganization.java | 39 +
.../resource/mgmt/OrganizationsCollection.java | 28 +-
.../usergrid/rest/test/security/TestUser.java | 5 +
.../resources/usergrid-custom-test.properties | 4 +-
.../org/apache/usergrid/ServiceApplication.java | 18 +-
.../org/apache/usergrid/ServiceITSuite.java | 2 +-
.../usergrid/services/ConnectionsServiceIT.java | 4 +-
.../usergrid/services/GroupServiceIT.java | 4 +-
.../usergrid/services/RolesServiceIT.java | 2 +-
.../usergrid/services/UsersServiceIT.java | 2 +-
.../AbstractServiceNotificationIT.java | 12 +-
.../notifications/NotifiersServiceIT.java | 2 +-
.../apns/NotificationsServiceIT.java | 132 +-
.../gcm/NotificationsServiceIT.java | 28 +-
.../resources/usergrid-custom-test.properties | 4 +-
stack/test-utils/pom.xml | 5 -
.../usergrid/cassandra/CassandraResource.java | 648 ++-------
.../cassandra/CassandraResourceTest.java | 7 +-
.../resources/usergrid-custom-test.properties | 2 +-
169 files changed, 5757 insertions(+), 3527 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/717eb15d/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 2d0673f,2cb01d4..8723968
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -2645,15 -2534,17 +2534,17 @@@ public class CpEntityManager implement
}
try {
- logger.debug("About to Write {}:{} version {}", new Object[] {
- cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
+ logger.debug( "About to Write {}:{} version {}", new Object[] {
+ cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion()
+ } );
- cpEntity = ecm.write( cpEntity ).toBlocking().last();
- cpEntity = ecm.write( cpEntity ).toBlocking().last();
++ cpEntity = ecm .write( cpEntity ).toBlocking().last();
- logger.debug("Wrote {}:{} version {}", new Object[] {
- cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
+ logger.debug( "Wrote {}:{} version {}", new Object[] {
+ cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion()
+ } );
- entityCache.put( new EntityScope( collectionScope, cpEntity.getId() ), cpEntity);
+ entityCache.put( new EntityScope( collectionScope, cpEntity.getId() ), cpEntity );
}
catch ( WriteUniqueVerifyException wuve ) {
handleWriteUniqueVerifyException( entity, wuve );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/717eb15d/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index 4268478,2d80672..2697958
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@@ -18,13 -19,9 +18,14 @@@ package org.apache.usergrid.corepersist
import com.google.inject.AbstractModule;
import com.google.inject.multibindings.Multibinder;
+ import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
import org.apache.usergrid.corepersistence.migration.GraphShardVersionMigration;
+import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
+import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
+import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+import org.apache.usergrid.persistence.collection.event.EntityDeleted;
+import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
+import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.guice.CollectionModule;
import org.apache.usergrid.persistence.core.guice.CommonModule;
import org.apache.usergrid.persistence.core.migration.data.DataMigration;
@@@ -53,19 -49,12 +54,20 @@@ public class GuiceModule extends Abstra
install(new MapModule());
install(new QueueModule());
- bind(CpEntityDeleteListener.class).asEagerSingleton();
- bind(CpEntityIndexDeleteListener.class).asEagerSingleton();
- bind(ManagerCache.class).to( CpManagerCache.class );
+ Multibinder<EntityDeleted> entityBinder
+ = Multibinder.newSetBinder(binder(), EntityDeleted.class);
+ entityBinder.addBinding().to(EntityDeletedHandler.class);
+
+ Multibinder<EntityVersionDeleted> versionBinder
+ = Multibinder.newSetBinder(binder(), EntityVersionDeleted.class);
+ versionBinder.addBinding().to(EntityVersionDeletedHandler.class);
+
+ Multibinder<EntityVersionCreated> versionCreatedMultibinder
+ = Multibinder.newSetBinder( binder(),EntityVersionCreated.class );
+ versionCreatedMultibinder.addBinding().to(EntityVersionCreatedHandler.class);
Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
+ dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/717eb15d/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/717eb15d/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index dff2f59,fa9f9df..95d77d0
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@@ -51,38 -56,14 +57,29 @@@ import static org.apache.usergrid.persi
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
- import org.junit.Before;
- import org.junit.After;
- import org.junit.Test;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
-
+//need to create system properties in test that can get applied
/**
* Test on read style clean-up of stale ElasticSearch indexes.
*/
public class StaleIndexCleanupTest extends AbstractCoreIT {
- private static final Logger logger = LoggerFactory.getLogger(StaleIndexCleanupTest.class );
+ private static final Logger logger = LoggerFactory.getLogger( StaleIndexCleanupTest.class );
- // take it easy on embedded Cassandra
- private static final long writeDelayMs = 50;
- private static final long readDelayMs = 50;
+ Lock sequential = new ReentrantLock();
+
+ @Before
+ public void before() {
+
+ // if tests run in parallel there will likely be a conflict over the allow.stale.entities
+ sequential.lock();
+ }
+
+ @After
+ public void after() {
+ System.clearProperty( "allow.stale.entities" );
+
+ }
/**
* Test that updating an entity causes the entity's version number to change.
@@@ -206,11 -196,10 +212,11 @@@
//last entities appear first
final Entity expected = maxVersions.get( index );
- assertEquals( "correct entity returned", expected, returned );
+ assertEquals("correct entity returned", expected, returned);
+
}
-
- } while ( cursor != null );
+ }
+ while ( cursor != null );
assertEquals( "Expect no stale candidates", numEntities, thingCount );
@@@ -227,142 -214,6 +231,143 @@@
/**
+ * Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an
+ * entity is deleted its old indexes are cleared from ElasticSearch.
+ */
+ @Test(timeout=10000)
+ public void testCleanupOnDelete() throws Exception {
+
+ logger.info("Started testStaleIndexCleanup()");
+
+ // turn off post processing stuff that cleans up stale entities
+ System.setProperty( "allow.stale.entities", "true" );
+
+ final EntityManager em = app.getEntityManager();
+
+ final int numEntities = 10;
+ final int numUpdates = 3;
+
+ // create lots of entities
+ final List<Entity> things = new ArrayList<Entity>(numEntities);
+ for ( int i=0; i<numEntities; i++) {
+ final String thingName = "thing" + i;
+ things.add( em.create("thing", new HashMap<String, Object>() {{
+ put("name", thingName);
+ }}));
+ Thread.sleep( writeDelayMs );
+ }
+ em.refreshIndex();
+
+ CandidateResults crs = queryCollectionCp( "things", "select *");
+ Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
+
+ // update each one a bunch of times
+ int count = 0;
+
+ List<Entity> maxVersions = new ArrayList<>(numEntities);
+
+ for ( Entity thing : things ) {
+ Entity toUpdate = null;
+
+ for ( int j=0; j<numUpdates; j++) {
+ toUpdate = em.get( thing.getUuid() );
+ toUpdate.setProperty( "property" + j, RandomStringUtils.randomAlphanumeric(10));
+
+ em.update(toUpdate);
+
+ Thread.sleep( writeDelayMs );
+ count++;
+ if ( count % 100 == 0 ) {
+ logger.info("Updated {} of {} times", count, numEntities * numUpdates);
+ }
+ }
+
+ maxVersions.add( toUpdate );
+ }
+ em.refreshIndex();
+
+ // query Core Persistence directly for total number of result candidates
+ crs = queryCollectionCp("things", "select *");
+ Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
+
+ // delete all entities
+ for ( Entity thing : things ) {
+ em.delete( thing );
+ }
+ em.refreshIndex();
+
+ // wait for indexes to be cleared for the deleted entities
+ count = 0;
+ do {
+ Thread.sleep(100);
+ crs = queryCollectionCp("things", "select *");
+ } while ( crs.size() > 0 && count++ < 14 );
+
+ Assert.assertEquals( "Expect no candidates", 0, crs.size() );
+ }
+
+
+ /**
+ * Test that the EntityDeleteImpl cleans up stale indexes on update. Ensures that when an
+ * entity is updated its old indexes are cleared from ElasticSearch.
+ */
+ @Test(timeout=10000)
+ public void testCleanupOnUpdate() throws Exception {
+
+ logger.info( "Started testCleanupOnUpdate()" );
+
+ final EntityManager em = app.getEntityManager();
+
+ final int numEntities = 10;
+ final int numUpdates = 3;
+
+ // create lots of entities
+ final List<Entity> things = new ArrayList<Entity>(numEntities);
+ for ( int i=0; i<numEntities; i++) {
+ final String thingName = "thing" + i;
+ things.add( em.create("thing", new HashMap<String, Object>() {{
+ put("name", thingName);
+ }}));
+ Thread.sleep( writeDelayMs );
+ }
+ em.refreshIndex();
+
+ CandidateResults crs = queryCollectionCp( "things", "select *");
+ Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
+
+ // update each one a bunch of times
+ int count = 0;
+
+ List<Entity> maxVersions = new ArrayList<>(numEntities);
+
+ for ( Entity thing : things ) {
+ Entity toUpdate = null;
+
+ for ( int j=0; j<numUpdates; j++) {
+ toUpdate = em.get( thing.getUuid() );
+ toUpdate.setProperty( "property" + j, RandomStringUtils.randomAlphanumeric(10));
+
+ em.update(toUpdate);
+
+ Thread.sleep( writeDelayMs );
+ count++;
+ if ( count % 100 == 0 ) {
+ logger.info("Updated {} of {} times", count, numEntities * numUpdates);
+ }
+ }
+
+ maxVersions.add( toUpdate );
+ }
+ em.refreshIndex();
+
+ // query Core Persistence directly for total number of result candidates
+ crs = queryCollectionCp("things", "select *");
+ Assert.assertEquals( "Expect candidates without earlier stale entities", numEntities, crs.size() );
+ }
+
+
+ /**
++ /**
* Go around EntityManager and get directly from Core Persistence.
*/
private org.apache.usergrid.persistence.model.entity.Entity getCpEntity( EntityRef eref ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/717eb15d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index b1e3b33,c4cdeeb..f69a64c
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@@ -258,9 -213,10 +251,13 @@@ public class EsEntityIndexBatchImpl imp
try {
responses = request.execute().actionGet();
+ } catch (Throwable t) {
+ logger.error("Unable to communicate with elasticsearch");
+ failureMonitor.fail("Unable to execute batch", t);
+ }
+ catch ( Throwable t ) {
+ log.error( "Unable to communicate with elasticsearch" );
+ failureMonitor.fail( "Unable to execute batch", t );
throw t;
}
@@@ -318,8 -293,8 +334,6 @@@
if ( !list.isEmpty() ) {
if ( list.get( 0 ) instanceof String ) {
-- Joiner joiner = Joiner.on( " " ).skipNulls();
-- String joined = joiner.join( list );
entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
new ArrayList( processCollectionForMap( list ) ) );
}
[38/50] incubator-usergrid git commit: Fix bug observed at test-time,
caused by new way of injecting EMF.
Posted by sf...@apache.org.
Fix bug observed at test-time, caused by new way of injecting EMF.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5f0dc347
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5f0dc347
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5f0dc347
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 5f0dc347711c0d7921669e20ec03723acfcf0c7c
Parents: c326179
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Nov 25 10:23:11 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Nov 25 10:23:11 2014 -0500
----------------------------------------------------------------------
.../corepersistence/events/EntityDeletedHandler.java | 4 +---
.../corepersistence/events/EntityVersionCreatedHandler.java | 8 ++++----
.../corepersistence/events/EntityVersionDeletedHandler.java | 5 +----
3 files changed, 6 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f0dc347/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index d34b51c..865e284 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -24,7 +24,6 @@ import org.apache.usergrid.persistence.model.entity.Id;
import java.util.UUID;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.slf4j.Logger;
@@ -52,8 +51,7 @@ public class EntityDeletedHandler implements EntityDeleted {
new Object[] { entityId.getType(), entityId.getUuid(), version,
scope.getName(), scope.getOwner(), scope.getApplication()});
- HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)emf;
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f0dc347/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 3462ecb..68c5547 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -22,7 +22,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
@@ -54,13 +53,14 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
new Object[] { entity.getId().getType(), entity.getId().getUuid(), entity.getVersion(),
scope.getName(), scope.getOwner(), scope.getApplication()});
- HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)emf;
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
- //TODO why aren't we using a collection fig here? This seems kludgy
+ // This check is for testing purposes and for a test that to be able to dynamically turn
+ // off and on delete previous versions so that it can test clean-up on read.
if ( System.getProperty( "allow.stale.entities", "false" ).equals( "false" )) {
+
ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5f0dc347/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index d63edb7..a6e971a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -18,7 +18,6 @@
package org.apache.usergrid.corepersistence.events;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
@@ -35,7 +34,6 @@ import rx.schedulers.Schedulers;
import java.util.List;
import com.google.inject.Inject;
-import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
import org.apache.usergrid.persistence.EntityManagerFactory;
@@ -59,8 +57,7 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
public void versionDeleted(
final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
- HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)emf;
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
[35/50] incubator-usergrid git commit: Fixing many more merge
problems.
Posted by sf...@apache.org.
Fixing many more merge problems.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2ee0eff3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2ee0eff3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2ee0eff3
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 2ee0eff3cd7a7b4bff53ed92cc411324634c9244
Parents: 3227693
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Nov 24 10:25:58 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Nov 24 10:25:58 2014 -0500
----------------------------------------------------------------------
.../usergrid/corepersistence/CpSetup.java | 16 +++---
.../usergrid/corepersistence/GuiceModule.java | 28 +++++++----
.../events/EntityDeletedHandler.java | 8 +--
.../events/EntityVersionCreatedHandler.java | 18 +++----
.../events/EntityVersionDeletedHandler.java | 7 ++-
.../corepersistence/StaleIndexCleanupTest.java | 51 ++++++++++++--------
.../corepersistence/TestGuiceModule.java | 12 +++--
.../usergrid/persistence/index/EntityIndex.java | 1 -
.../apache/usergrid/rest/TestContextSetup.java | 6 +--
.../collection/DuplicateNameIT.java | 2 +-
.../rest/test/resource/TestContext.java | 3 +-
11 files changed, 90 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index d48a20c..35ef63b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@ -62,15 +62,11 @@ public class CpSetup implements Setup {
private static final Logger logger = LoggerFactory.getLogger( CpSetup.class );
private static Injector injector = null;
+ private static EntityManagerFactory emf;
-
- private final org.apache.usergrid.persistence.EntityManagerFactory emf;
private final CassandraService cass;
-
-
-
/**
* Instantiates a new setup object.
*
@@ -82,11 +78,9 @@ public class CpSetup implements Setup {
}
-
-
public static Injector getInjector() {
if ( injector == null ) {
- injector = Guice.createInjector( new GuiceModule() );
+ injector = Guice.createInjector( new GuiceModule( emf ) );
}
return injector;
}
@@ -175,14 +169,16 @@ public class CpSetup implements Setup {
emf.initializeApplication( DEFAULT_ORGANIZATION,
emf.getDefaultAppId(), DEFAULT_APPLICATION, null );
} catch (ApplicationAlreadyExistsException ex) {
- logger.warn("Application {}/{} already exists", DEFAULT_ORGANIZATION, DEFAULT_APPLICATION);
+ logger.warn("Application {}/{} already exists",
+ DEFAULT_ORGANIZATION, DEFAULT_APPLICATION);
}
try {
emf.initializeApplication( DEFAULT_ORGANIZATION,
emf.getManagementAppId(), MANAGEMENT_APPLICATION, null );
} catch (ApplicationAlreadyExistsException ex) {
- logger.warn("Application {}/{} already exists", DEFAULT_ORGANIZATION, MANAGEMENT_APPLICATION);
+ logger.warn("Application {}/{} already exists",
+ DEFAULT_ORGANIZATION, MANAGEMENT_APPLICATION);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index 2697958..98f473a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@ -23,6 +23,7 @@ import org.apache.usergrid.corepersistence.migration.GraphShardVersionMigration;
import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
@@ -44,9 +45,19 @@ public class GuiceModule extends AbstractModule {
private static final Logger logger = LoggerFactory.getLogger(GuiceModule.class);
+ private EntityManagerFactory emf;
+
+ GuiceModule( EntityManagerFactory emf ) {
+ this.emf = emf;
+ }
+
@Override
protected void configure() {
+ if ( emf != null ) {
+ bind( EntityManagerFactory.class ).toInstance( emf );
+ }
+
install(new CommonModule());
install(new CollectionModule());
install(new GraphModule());
@@ -54,23 +65,22 @@ public class GuiceModule extends AbstractModule {
install(new MapModule());
install(new QueueModule());
- Multibinder<EntityDeleted> entityBinder
- = Multibinder.newSetBinder(binder(), EntityDeleted.class);
+ Multibinder<EntityDeleted> entityBinder =
+ Multibinder.newSetBinder(binder(), EntityDeleted.class);
entityBinder.addBinding().to(EntityDeletedHandler.class);
- Multibinder<EntityVersionDeleted> versionBinder
- = Multibinder.newSetBinder(binder(), EntityVersionDeleted.class);
+ Multibinder<EntityVersionDeleted> versionBinder =
+ Multibinder.newSetBinder(binder(), EntityVersionDeleted.class);
versionBinder.addBinding().to(EntityVersionDeletedHandler.class);
- Multibinder<EntityVersionCreated> versionCreatedMultibinder
- = Multibinder.newSetBinder( binder(),EntityVersionCreated.class );
+ Multibinder<EntityVersionCreated> versionCreatedMultibinder =
+ Multibinder.newSetBinder( binder(), EntityVersionCreated.class );
versionCreatedMultibinder.addBinding().to(EntityVersionCreatedHandler.class);
- Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
+ Multibinder<DataMigration> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), DataMigration.class );
dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
-
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index bd738fe..d34b51c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -17,15 +17,15 @@
*/
package org.apache.usergrid.corepersistence.events;
+import com.google.inject.Inject;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.event.EntityDeleted;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.model.entity.Id;
import java.util.UUID;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
+import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -37,6 +37,8 @@ import org.slf4j.LoggerFactory;
public class EntityDeletedHandler implements EntityDeleted {
private static final Logger logger = LoggerFactory.getLogger(EntityDeletedHandler.class );
+ @Inject
+ EntityManagerFactory emf;
public EntityDeletedHandler() {
logger.debug("Created");
@@ -50,7 +52,7 @@ public class EntityDeletedHandler implements EntityDeleted {
new Object[] { entityId.getType(), entityId.getUuid(), version,
scope.getName(), scope.getOwner(), scope.getApplication()});
- HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+ HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)emf;
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 08b8fbd..3462ecb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -17,33 +17,36 @@
*/
package org.apache.usergrid.corepersistence.events;
+import com.google.inject.Inject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
-import org.apache.usergrid.corepersistence.CpSetup;
import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
+import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.model.entity.Entity;
-
/**
* Clean up stale entity indexes when new version of Entity created. Called when an Entity is
* updated by the Collections module and we react by calling the Query Index module and removing
* any indexes that exist for previous versions of the the Entity.
*/
public class EntityVersionCreatedHandler implements EntityVersionCreated {
-
private static final Logger logger = LoggerFactory.getLogger(EntityVersionCreatedHandler.class );
+ @Inject
+ EntityManagerFactory emf;
+
+
public EntityVersionCreatedHandler() {
logger.debug("EntityVersionCreated");
}
+
@Override
public void versionCreated( final CollectionScope scope, final Entity entity ) {
logger.debug("Entering deleted for entity {}:{} v {} "
@@ -51,17 +54,14 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
new Object[] { entity.getId().getType(), entity.getId().getUuid(), entity.getVersion(),
scope.getName(), scope.getOwner(), scope.getApplication()});
- HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+ HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)emf;
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
- EntityIndexBatch batch = ei.createBatch();
-
//TODO why aren't we using a collection fig here? This seems kludgy
if ( System.getProperty( "allow.stale.entities", "false" ).equals( "false" )) {
- batch.deindexPreviousVersions( entity );
- batch.execute();
+ ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 046e9c2..d63edb7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -36,6 +36,7 @@ import java.util.List;
import com.google.inject.Inject;
import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
+import org.apache.usergrid.persistence.EntityManagerFactory;
/**
@@ -47,8 +48,10 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
private final SerializationFig serializationFig;
+ private EntityManagerFactory emf;
+
@Inject
- public EntityVersionDeletedHandler(SerializationFig fig) {
+ public EntityVersionDeletedHandler(SerializationFig fig, EntityManagerFactory emf) {
this.serializationFig = fig;
}
@@ -56,7 +59,7 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
public void versionDeleted(
final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
- HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+ HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)emf;
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 95d77d0..e1c0214 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -35,7 +35,6 @@ import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.Results;
-import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
@@ -52,10 +51,14 @@ import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import com.fasterxml.uuid.UUIDComparator;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+import org.junit.After;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import org.junit.Before;
//need to create system properties in test that can get applied
@@ -65,6 +68,9 @@ import static org.junit.Assert.assertTrue;
public class StaleIndexCleanupTest extends AbstractCoreIT {
private static final Logger logger = LoggerFactory.getLogger( StaleIndexCleanupTest.class );
+ // take it easy on Cassandra
+ private static final long writeDelayMs = 50;
+ private static final long readDelayMs = 50;
Lock sequential = new ReentrantLock();
@@ -108,14 +114,16 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
assertEquals( "widget", cpUpdated.getField( "stuff" ).getValue() );
UUID newVersion = cpUpdated.getVersion();
- assertTrue( "New version is greater than old", UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
+ assertTrue( "New version is greater than old",
+ UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
assertEquals( 2, queryCollectionCp( "things", "thing", "select *" ).size() );
}
/**
- * Test that the CpRelationManager cleans up and stale indexes that it finds when it is building search results.
+ * Test that the CpRelationManager cleans up and stale indexes that it finds when
+ * it is building search results.
*/
@Test
public void testStaleIndexCleanup() throws Exception {
@@ -185,8 +193,8 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
// should return numEntities becuase it filters out the stale entities
final int limit = 8;
- //we order by updateCount asc, this forces old versions to appear first, otherwise, we don't clean them up in
- // our versions
+ // we order by updateCount asc, this forces old versions to appear first, otherwise,
+ // we don't clean them up in our versions
Query q = Query.fromQL( "select * order by updateCount asc" );
q.setLimit( limit );
int thingCount = 0;
@@ -258,7 +266,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
em.refreshIndex();
- CandidateResults crs = queryCollectionCp( "things", "select *");
+ CandidateResults crs = queryCollectionCp( "things", "thing", "select *");
Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
// update each one a bunch of times
@@ -287,7 +295,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
em.refreshIndex();
// query Core Persistence directly for total number of result candidates
- crs = queryCollectionCp("things", "select *");
+ crs = queryCollectionCp("things", "thing", "select *");
Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
// delete all entities
@@ -300,7 +308,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
count = 0;
do {
Thread.sleep(100);
- crs = queryCollectionCp("things", "select *");
+ crs = queryCollectionCp("things", "thing", "select *");
} while ( crs.size() > 0 && count++ < 14 );
Assert.assertEquals( "Expect no candidates", 0, crs.size() );
@@ -332,7 +340,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
em.refreshIndex();
- CandidateResults crs = queryCollectionCp( "things", "select *");
+ CandidateResults crs = queryCollectionCp( "things", "things", "select *");
Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
// update each one a bunch of times
@@ -361,7 +369,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
em.refreshIndex();
// query Core Persistence directly for total number of result candidates
- crs = queryCollectionCp("things", "select *");
+ crs = queryCollectionCp("things", "thing", "select *");
Assert.assertEquals( "Expect candidates without earlier stale entities", numEntities, crs.size() );
}
@@ -374,29 +382,34 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
EntityManager em = app.getEntityManager();
- CollectionScope cs = new CollectionScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
- new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
- CpNamingUtils.getCollectionScopeNameFromEntityType( eref.getType() ) );
+ CollectionScope cs = new CollectionScopeImpl(
+ new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
+ new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
+ CpNamingUtils.getCollectionScopeNameFromEntityType( eref.getType() ) );
- EntityCollectionManagerFactory ecmf = CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class );
+ EntityCollectionManagerFactory ecmf =
+ CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class );
EntityCollectionManager ecm = ecmf.createCollectionManager( cs );
- return ecm.load( new SimpleId( eref.getUuid(), eref.getType() ) ).toBlocking().lastOrDefault( null );
+ return ecm.load( new SimpleId( eref.getUuid(), eref.getType() ) )
+ .toBlocking().lastOrDefault( null );
}
/**
- * Go around EntityManager and execute query directly against Core Persistence. Results may include stale index
- * entries.
+ * Go around EntityManager and execute query directly against Core Persistence.
+ * Results may include stale index entries.
*/
- private CandidateResults queryCollectionCp( final String collName, final String type, final String query ) {
+ private CandidateResults queryCollectionCp(
+ final String collName, final String type, final String query ) {
EntityManager em = app.getEntityManager();
EntityIndexFactory eif = CpSetup.getInjector().getInstance( EntityIndexFactory.class );
- ApplicationScope as = new ApplicationScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ) );
+ ApplicationScope as = new ApplicationScopeImpl(
+ new SimpleId( em.getApplicationId(), TYPE_APPLICATION ) );
EntityIndex ei = eif.createEntityIndex( as );
IndexScope is = new IndexScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
index 7d1bd69..9a0b195 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
@@ -19,15 +19,19 @@
package org.apache.usergrid.corepersistence;
import com.google.inject.AbstractModule;
+import org.apache.usergrid.persistence.EntityManagerFactory;
public class TestGuiceModule extends AbstractModule {
- @Override
- protected void configure() {
+ private static EntityManagerFactory emf;
- install( new GuiceModule() );
+ public TestGuiceModule( EntityManagerFactory emf ) {
+ this.emf = emf;
}
-
+ @Override
+ protected void configure() {
+ install( new GuiceModule( emf ) );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 29d8015..6d2cdd2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -24,7 +24,6 @@ import java.util.UUID;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.CandidateResults;
-import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java b/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java
index 2da1033..4b7cc69 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java
@@ -22,7 +22,6 @@ import org.junit.runner.Description;
import org.junit.runners.model.Statement;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
-import org.apache.usergrid.rest.management.OrganizationsIT;
import org.apache.usergrid.rest.test.resource.TestContext;
import org.apache.usergrid.rest.test.security.TestAdminUser;
@@ -31,8 +30,9 @@ import java.io.IOException;
/**
- * A self configuring TestContext which sets itself up it implements TestRule. With a @Rule annotation, an instance of
- * this Class as a public member in any test class or abstract test class will auto svcSetup itself before each test.
+ * A self configuring TestContext which sets itself up it implements TestRule. With a @Rule
+ * annotation, an instance of this Class as a public member in any test class or abstract
+ * test class will auto svcSetup itself before each test.
*/
public class TestContextSetup extends TestContext implements TestRule {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java
index 39344d0..5c98785 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java
@@ -55,7 +55,7 @@ public class DuplicateNameIT extends AbstractRestIT {
refreshIndex( context.getAppUuid() );
- Injector injector = Guice.createInjector( new TestGuiceModule() );
+ Injector injector = Guice.createInjector( new TestGuiceModule( null ) );
SerializationFig sfig = injector.getInstance( SerializationFig.class );
// wait for any temporary unique value records to timeout
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ee0eff3/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/TestContext.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/TestContext.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/TestContext.java
index 24a05e7..814dd35 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/TestContext.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/TestContext.java
@@ -124,7 +124,8 @@ public class TestContext {
}
- /** Create the app if it doesn't exist with the given TestUser. If the app exists, the user is logged in */
+ /** Create the app if it doesn't exist with the given TestUser.
+ * If the app exists, the user is logged in */
public TestContext loginUser() {
// nothing to do
if ( activeUser.isLoggedIn() ) {
[45/50] incubator-usergrid git commit: Fixes to entity created,
version created and delete event handlers.
Posted by sf...@apache.org.
Fixes to entity created, version created and delete event handlers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7f1d533b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7f1d533b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7f1d533b
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 7f1d533b6ced46307ae76c304fc0efc053e746ca
Parents: bf47eb3
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Dec 10 09:27:06 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Dec 10 09:27:06 2014 -0500
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 52 ++------------------
.../corepersistence/CpRelationManager.java | 33 ++++++++-----
.../usergrid/corepersistence/GuiceModule.java | 7 ++-
.../events/EntityDeletedHandler.java | 22 ++++++---
.../events/EntityVersionCreatedHandler.java | 34 +++++++------
.../events/EntityVersionDeletedHandler.java | 30 ++++++++---
.../results/FilteringLoader.java | 7 ++-
.../corepersistence/StaleIndexCleanupTest.java | 43 +++++++++++-----
.../usergrid/persistence/CollectionIT.java | 15 ++++--
.../index/impl/EsEntityIndexBatchImpl.java | 6 ++-
.../index/impl/EsEntityIndexImpl.java | 25 +++++-----
11 files changed, 149 insertions(+), 125 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index e1b0a73..3ab767c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -607,50 +607,6 @@ public class CpEntityManager implements EntityManager {
if ( entity != null ) {
- // first, delete entity in every collection and connection scope in which it is indexed
-
-// RelationManager rm = getRelationManager( entityRef );
-// Map<String, Map<UUID, Set<String>>> owners = rm.getOwners();
-//
-// logger.debug( "Deleting indexes of all {} collections owning the entity",
-// owners.keySet().size() );
-//
-// final EntityIndex ei = managerCache.getEntityIndex(getApplicationScope());
-//
-// final EntityIndexBatch batch = ei.createBatch();
-//
-// for ( String ownerType : owners.keySet() ) {
-// Map<UUID, Set<String>> collectionsByUuid = owners.get( ownerType );
-//
-// for ( UUID uuid : collectionsByUuid.keySet() ) {
-// Set<String> collectionNames = collectionsByUuid.get( uuid );
-// for ( String coll : collectionNames ) {
-//
-// IndexScope indexScope = new IndexScopeImpl(
-// new SimpleId( uuid, ownerType ),
-// CpNamingUtils.getCollectionScopeNameFromCollectionName( coll ) );
-//
-//
-// batch.index( indexScope, entity );
-// }
-// }
-// }
-//
-// // deindex from default index scope
-// IndexScope defaultIndexScope = new IndexScopeImpl(
-// getApplicationScope().getApplication(),
-// CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
-//
-// batch.deindex(defaultIndexScope, entity );
-//
-// IndexScope allTypesIndexScope = new IndexScopeImpl(
-// getApplicationScope().getApplication(),
-// CpNamingUtils.ALL_TYPES);
-//
-// batch.deindex( allTypesIndexScope, entity );
-//
-// batch.execute();
-
decrementEntityCollection( Schema.defaultCollectionName( entityId.getType() ) );
// and finally...
@@ -2010,10 +1966,10 @@ public class CpEntityManager implements EntityManager {
for ( Query.CounterFilterPredicate filter : filters ) {
CounterUtils.AggregateCounterSelection selection =
- new CounterUtils.AggregateCounterSelection( filter.getName(),
- getUuid( getUserByIdentifier( filter.getUser() ) ),
- getUuid( getGroupByIdentifier( filter.getGroup() ) ),
- org.apache.usergrid.mq.Queue.getQueueId( filter.getQueue() ), filter.getCategory() );
+ new CounterUtils.AggregateCounterSelection( filter.getName(),
+ getUuid( getUserByIdentifier( filter.getUser() ) ),
+ getUuid( getGroupByIdentifier( filter.getGroup() ) ),
+ org.apache.usergrid.mq.Queue.getQueueId( filter.getQueue() ), filter.getCategory() );
selections.put( selection.getRow( resolution ), selection );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index dcb4ba1..135688e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -220,7 +220,8 @@ public class CpRelationManager implements RelationManager {
this.indexBucketLocator = indexBucketLocator; // TODO: this also
// load the Core Persistence version of the head entity as well
- this.headEntityScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), headEntity.getType());
+ this.headEntityScope = getCollectionScopeNameFromEntityType(
+ applicationScope.getApplication(), headEntity.getType());
if ( logger.isDebugEnabled() ) {
logger.debug( "Loading head entity {}:{} from scope\n app {}\n owner {}\n name {}",
@@ -410,16 +411,16 @@ public class CpRelationManager implements RelationManager {
if ( CpNamingUtils.isCollectionEdgeType( edge.getType() ) ) {
String collName = CpNamingUtils.getCollectionName( edge.getType() );
- indexScope =
- new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(), sourceEntity.getType() ),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
+ indexScope = new IndexScopeImpl(
+ new SimpleId( sourceEntity.getUuid(), sourceEntity.getType()),
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
}
else {
String connName = CpNamingUtils.getConnectionType( edge.getType() );
- indexScope =
- new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(), sourceEntity.getType() ),
- CpNamingUtils.getConnectionScopeName( connName ) );
+ indexScope = new IndexScopeImpl(
+ new SimpleId( sourceEntity.getUuid(), sourceEntity.getType() ),
+ CpNamingUtils.getConnectionScopeName( connName ) );
}
entityIndexBatch.index( indexScope, cpEntity );
@@ -604,7 +605,8 @@ public class CpRelationManager implements RelationManager {
public Entity addToCollection( String collName, EntityRef itemRef, boolean connectBack )
throws Exception {
- CollectionScope memberScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), itemRef.getType());
+ CollectionScope memberScope = getCollectionScopeNameFromEntityType(
+ applicationScope.getApplication(), itemRef.getType());
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
org.apache.usergrid.persistence.model.entity.Entity memberEntity =
@@ -637,7 +639,8 @@ public class CpRelationManager implements RelationManager {
}
// load the new member entity to be added to the collection from its default scope
- CollectionScope memberScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), itemRef.getType());
+ CollectionScope memberScope = getCollectionScopeNameFromEntityType(
+ applicationScope.getApplication(), itemRef.getType());
//TODO, this double load should disappear once events are in
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
@@ -789,7 +792,8 @@ public class CpRelationManager implements RelationManager {
}
// load the entity to be removed to the collection
- CollectionScope memberScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), itemRef.getType());
+ CollectionScope memberScope = getCollectionScopeNameFromEntityType(
+ applicationScope.getApplication(), itemRef.getType());
if ( logger.isDebugEnabled() ) {
logger.debug( "Loading entity to remove from collection "
@@ -1005,7 +1009,8 @@ public class CpRelationManager implements RelationManager {
ConnectionRefImpl connection = new ConnectionRefImpl( headEntity, connectionType, connectedEntityRef );
- CollectionScope targetScope = getCollectionScopeNameFromEntityType(applicationScope.getApplication(), connectedEntityRef.getType());
+ CollectionScope targetScope = getCollectionScopeNameFromEntityType(
+ applicationScope.getApplication(), connectedEntityRef.getType());
if ( logger.isDebugEnabled() ) {
logger.debug("createConnection(): "
@@ -1333,7 +1338,8 @@ public class CpRelationManager implements RelationManager {
headEntity = em.validate( headEntity );
- IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(), CpNamingUtils.getConnectionScopeName( connectionType ) );
+ IndexScope indexScope = new IndexScopeImpl(
+ cpHeadEntity.getId(), CpNamingUtils.getConnectionScopeName( connectionType ) );
final SearchTypes searchTypes = SearchTypes.fromNullableTypes( connectedEntityType );
@@ -1564,7 +1570,8 @@ public class CpRelationManager implements RelationManager {
* @param crs Candidates to be considered for results
* @param collName Name of collection or null if querying all types
*/
- private Results buildResults( final IndexScope indexScope, final Query query, final CandidateResults crs, final String collName ) {
+ private Results buildResults( final IndexScope indexScope, final Query query,
+ final CandidateResults crs, final String collName ) {
logger.debug( "buildResults() for {} from {} candidates", collName, crs.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
index 2ff6c4f..ed6cba2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/GuiceModule.java
@@ -48,6 +48,8 @@ public class GuiceModule extends AbstractModule {
private EntityManagerFactory emf;
+ public static final String EVENTS_DISABLED = "corepersistence.events.disabled";
+
GuiceModule( EntityManagerFactory emf ) {
this.emf = emf;
}
@@ -68,10 +70,11 @@ public class GuiceModule extends AbstractModule {
bind(ManagerCache.class).to( CpManagerCache.class );
- Multibinder<DataMigration> dataMigrationMultibinder = Multibinder.newSetBinder( binder(), DataMigration.class );
+ Multibinder<DataMigration> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), DataMigration.class );
dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
- //dataMigrationMultibinder.addBinding().to( EntityDataMigration.class );
+ dataMigrationMultibinder.addBinding().to( EntityDataMigration.class );
Multibinder<EntityDeleted> entityBinder =
Multibinder.newSetBinder(binder(), EntityDeleted.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
index 865e284..7d2ab2d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedHandler.java
@@ -24,6 +24,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
import java.util.UUID;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
+import static org.apache.usergrid.corepersistence.GuiceModule.EVENTS_DISABLED;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.slf4j.Logger;
@@ -39,20 +40,27 @@ public class EntityDeletedHandler implements EntityDeleted {
@Inject
EntityManagerFactory emf;
- public EntityDeletedHandler() {
- logger.debug("Created");
- }
@Override
public void deleted(CollectionScope scope, Id entityId, UUID version) {
- logger.debug("Entering deleted for entity {}:{} v {} "
+ // This check is for testing purposes and for a test that to be able to dynamically turn
+ // off and on delete previous versions so that it can test clean-up on read.
+ if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" )) {
+ return;
+ }
+
+ logger.debug("Handling deleted event for entity {}:{} v {} "
+ "scope\n name: {}\n owner: {}\n app: {}",
- new Object[] { entityId.getType(), entityId.getUuid(), version,
- scope.getName(), scope.getOwner(), scope.getApplication()});
+ new Object[] {
+ entityId.getType(),
+ entityId.getUuid(),
+ version,
+ scope.getName(),
+ scope.getOwner(),
+ scope.getApplication()});
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
-
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
ei.deleteAllVersionsOfEntity( entityId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
index 68c5547..a797fc2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionCreatedHandler.java
@@ -22,6 +22,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
+import static org.apache.usergrid.corepersistence.GuiceModule.EVENTS_DISABLED;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
@@ -41,27 +42,28 @@ public class EntityVersionCreatedHandler implements EntityVersionCreated {
EntityManagerFactory emf;
- public EntityVersionCreatedHandler() {
- logger.debug("EntityVersionCreated");
- }
-
-
@Override
public void versionCreated( final CollectionScope scope, final Entity entity ) {
- logger.debug("Entering deleted for entity {}:{} v {} "
- + "scope\n name: {}\n owner: {}\n app: {}",
- new Object[] { entity.getId().getType(), entity.getId().getUuid(), entity.getVersion(),
- scope.getName(), scope.getOwner(), scope.getApplication()});
-
- CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
-
- final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
// This check is for testing purposes and for a test that to be able to dynamically turn
// off and on delete previous versions so that it can test clean-up on read.
- if ( System.getProperty( "allow.stale.entities", "false" ).equals( "false" )) {
-
- ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
+ if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" )) {
+ return;
}
+
+ logger.debug("Handling versionCreated for entity {}:{} v {} "
+ + "scope\n name: {}\n owner: {}\n app: {}",
+ new Object[] {
+ entity.getId().getType(),
+ entity.getId().getUuid(),
+ entity.getVersion(),
+ scope.getName(),
+ scope.getOwner(),
+ scope.getApplication()});
+
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+
+ ei.deletePreviousVersions( entity.getId(), entity.getVersion() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index 23097c4..bfffc26 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@ -19,8 +19,8 @@ package org.apache.usergrid.corepersistence.events;
import com.google.inject.Inject;
import java.util.List;
-import java.util.UUID;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
+import static org.apache.usergrid.corepersistence.GuiceModule.EVENTS_DISABLED;
import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.MvccEntity;
@@ -32,6 +32,8 @@ import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -42,20 +44,35 @@ import rx.schedulers.Schedulers;
* If we do need it then it should be wired in via GuiceModule in the corepersistence package.
*/
public class EntityVersionDeletedHandler implements EntityVersionDeleted {
+ private static final Logger logger = LoggerFactory.getLogger(EntityVersionDeletedHandler.class );
- private final SerializationFig serializationFig;
+ @Inject
+ private SerializationFig serializationFig;
+ @Inject
private EntityManagerFactory emf;
- @Inject
- public EntityVersionDeletedHandler(SerializationFig fig, EntityManagerFactory emf) {
- this.serializationFig = fig;
- }
@Override
public void versionDeleted(
final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
+ // This check is for testing purposes and for a test that to be able to dynamically turn
+ // off and on delete previous versions so that it can test clean-up on read.
+ if ( System.getProperty( EVENTS_DISABLED, "false" ).equals( "true" )) {
+ return;
+ }
+
+ logger.debug("Handling versionDeleted count={} event for entity {}:{} v {} "
+ + "scope\n name: {}\n owner: {}\n app: {}",
+ new Object[] {
+ entityVersions.size(),
+ entityId.getType(),
+ entityId.getUuid(),
+ scope.getName(),
+ scope.getOwner(),
+ scope.getApplication()});
+
CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
@@ -66,6 +83,7 @@ public class EntityVersionDeletedHandler implements EntityVersionDeleted {
new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
scope.getName()
);
+
rx.Observable.from(entityVersions)
.subscribeOn(Schedulers.io())
.buffer(serializationFig.getBufferSize())
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index c1d42ea..dca59e0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -71,7 +71,12 @@ public class FilteringLoader implements ResultsLoader {
* @param applicationScope The application scope to perform the load
* @param indexScope The index scope used in the search
*/
- protected FilteringLoader( final ManagerCache managerCache, final ResultsVerifier resultsVerifier, final ApplicationScope applicationScope, final IndexScope indexScope ) {
+ protected FilteringLoader(
+ final ManagerCache managerCache,
+ final ResultsVerifier resultsVerifier,
+ final ApplicationScope applicationScope,
+ final IndexScope indexScope ) {
+
this.managerCache = managerCache;
this.resultsVerifier = resultsVerifier;
this.applicationScope = applicationScope;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index f159146..5b9db6e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -38,7 +38,6 @@ import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.index.EntityIndex;
@@ -53,6 +52,7 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import com.fasterxml.uuid.UUIDComparator;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import static org.apache.usergrid.corepersistence.GuiceModule.EVENTS_DISABLED;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
@@ -62,7 +62,6 @@ import static org.junit.Assert.assertTrue;
import org.junit.Before;
-//need to create system properties in test that can get applied
/**
* Test on read style clean-up of stale ElasticSearch indexes.
*/
@@ -84,7 +83,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
@After
public void after() {
- System.clearProperty( "allow.stale.entities" );
+ System.clearProperty( EVENTS_DISABLED );
}
@@ -94,6 +93,9 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
@Test
public void testUpdateVersioning() throws Exception {
+ // turn off post processing stuff that cleans up stale entities
+ System.setProperty( EVENTS_DISABLED, "true" );
+
final EntityManager em = app.getEntityManager();
Entity thing = em.create( "thing", new HashMap<String, Object>() {{
@@ -131,7 +133,8 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
logger.info( "Started testStaleIndexCleanup()" );
- // TODO: turn off post processing stuff that cleans up stale entities
+ // turn off post processing stuff that cleans up stale entities
+ System.setProperty( EVENTS_DISABLED, "true" );
final EntityManager em = app.getEntityManager();
@@ -160,9 +163,6 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
List<Entity> maxVersions = new ArrayList<>( numEntities );
-
-
-
for ( Entity thing : things ) {
Entity toUpdate = null;
@@ -174,7 +174,6 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
toUpdate.setProperty( "updateCount", updateCount.getAndIncrement() );
em.update( toUpdate );
-
count++;
if ( count % 100 == 0 ) {
logger.info( "Updated {} of {} times", count, numEntities * numUpdates );
@@ -249,7 +248,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
logger.info("Started testStaleIndexCleanup()");
// turn off post processing stuff that cleans up stale entities
- System.setProperty( "allow.stale.entities", "true" );
+ System.setProperty( EVENTS_DISABLED, "true" );
final EntityManager em = app.getEntityManager();
@@ -299,18 +298,21 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
crs = queryCollectionCp("things", "thing", "select *");
Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
+ // turn ON post processing stuff that cleans up stale entities
+ System.setProperty( EVENTS_DISABLED, "false" );
+
// delete all entities
for ( Entity thing : things ) {
em.delete( thing );
}
- em.refreshIndex();
// wait for indexes to be cleared for the deleted entities
count = 0;
do {
Thread.sleep(100);
crs = queryCollectionCp("things", "thing", "select *");
- } while ( crs.size() > 0 && count++ < 14 );
+ em.refreshIndex();
+ } while ( crs.size() > 0 && count++ < 15 );
Assert.assertEquals( "Expect no candidates", 0, crs.size() );
}
@@ -325,6 +327,9 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
logger.info( "Started testCleanupOnUpdate()" );
+ // turn off post processing stuff that cleans up stale entities
+ System.setProperty( EVENTS_DISABLED, "true" );
+
final EntityManager em = app.getEntityManager();
final int numEntities = 10;
@@ -341,9 +346,12 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
em.refreshIndex();
- CandidateResults crs = queryCollectionCp( "things", "things", "select *");
+ CandidateResults crs = queryCollectionCp( "things", "thing", "select *");
Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
+ // turn off post processing stuff that cleans up stale entities
+ System.setProperty( EVENTS_DISABLED, "false" );
+
// update each one a bunch of times
int count = 0;
@@ -369,6 +377,14 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
em.refreshIndex();
+ // wait for indexes to be cleared for the deleted entities
+ count = 0;
+ do {
+ Thread.sleep(100);
+ crs = queryCollectionCp("things", "thing", "select *");
+ em.refreshIndex();
+ } while ( crs.size() > 0 && count++ < 15 );
+
// query Core Persistence directly for total number of result candidates
crs = queryCollectionCp("things", "thing", "select *");
Assert.assertEquals( "Expect candidates without earlier stale entities", numEntities, crs.size() );
@@ -383,7 +399,8 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
EntityManager em = app.getEntityManager();
- CollectionScope cs = getCollectionScopeNameFromEntityType( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ), eref.getType() );
+ CollectionScope cs = getCollectionScopeNameFromEntityType(
+ new SimpleId( em.getApplicationId(), TYPE_APPLICATION ), eref.getType() );
EntityCollectionManagerFactory ecmf =
CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
index 23983e1..b24bd31 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionIT.java
@@ -934,10 +934,10 @@ public class
EntityManager em = app.getEntityManager();
assertNotNull( em );
- int size = 20;
+ int initialSize = 20;
List<UUID> entityIds = new ArrayList<UUID>();
- for ( int i = 0; i < size; i++ ) {
+ for ( int i = 0; i < initialSize; i++ ) {
Map<String, Object> properties = new LinkedHashMap<String, Object>();
properties.put( "name", "object" + i );
Entity created = em.create( "objects", properties );
@@ -954,28 +954,33 @@ public class
LOG.info( JsonUtils.mapToFormattedJsonString( r.getEntities() ) );
- assertEquals( size, r.size() );
+ assertEquals(initialSize, r.size() );
// check they're all the same before deletion
- for ( int i = 0; i < size; i++ ) {
+ for ( int i = 0; i < initialSize; i++ ) {
assertEquals( entityIds.get( i ), r.getEntities().get( i ).getUuid() );
}
// now delete 5 items that will span the 10 pages
+ int numDeleted = 0;
for ( int i = 5; i < 10; i++ ) {
Entity entity = r.getEntities().get( i );
em.delete( entity );
entityIds.remove( entity.getUuid() );
+ numDeleted++;
}
em.refreshIndex();
+ // wait for indexes to be cleared
+ Thread.sleep( 500 );
+
// now query with paging
query = new Query();
r = em.searchCollection( em.getApplicationRef(), "objects", query );
- assertEquals( 10, r.size() );
+ assertEquals( query.getLimit(), r.size() );
for ( int i = 0; i < 10; i++ ) {
assertEquals( entityIds.get( i ), r.getEntities().get( i ).getUuid() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 2f555ed..6c049fc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -143,7 +143,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
- bulkRequest.add( client.prepareIndex(alias.getWriteAlias(), entityType, indexId ).setSource( entityAsMap ) );
+ bulkRequest.add( client.prepareIndex(
+ alias.getWriteAlias(), entityType, indexId ).setSource( entityAsMap ) );
maybeFlush();
@@ -181,7 +182,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
log.debug( "De-indexing type {} with documentId '{}'", entityType, indexId );
- bulkRequest.add( client.prepareDelete(alias.getWriteAlias(), entityType, indexId ).setRefresh( refresh ) );
+ bulkRequest.add( client.prepareDelete(
+ alias.getWriteAlias(), entityType, indexId ).setRefresh( refresh ) );
log.debug( "Deindexed Entity with index id " + indexId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7f1d533b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 51ee915..1ae2d6a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -233,7 +233,7 @@ public class EsEntityIndexImpl implements EntityIndex {
// to receive documents. Occasionally we see errors.
// See this post: http://s.apache.org/index-missing-exception
- logger.info( "Refreshing Created new Index Name [{}]", alias);
+ logger.debug( "Testing new index name [{}]", alias);
final RetryOperation retryOperation = new RetryOperation() {
@Override
@@ -438,9 +438,6 @@ public class EsEntityIndexImpl implements EntityIndex {
public void refresh() {
-
- logger.info( "Refreshing Created new Index Name [{}]", alias);
-
final RetryOperation retryOperation = new RetryOperation() {
@Override
public boolean doOp() {
@@ -452,7 +449,7 @@ public class EsEntityIndexImpl implements EntityIndex {
return true;
}
catch ( IndexMissingException e ) {
- logger.error( "Unable to refresh index after create. Waiting before sleeping.", e );
+ logger.error( "Unable to refresh index. Waiting before sleeping.", e );
throw e;
}
}
@@ -510,8 +507,9 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
public void deleteAllVersionsOfEntity( Id entityId ) {
- final TermQueryBuilder tqb = QueryBuilders.termQuery(
- ENTITYID_ID_FIELDNAME, entityId.getUuid().toString().toLowerCase() );
+ String idString = IndexingUtils.idString( entityId ).toLowerCase();
+
+ final TermQueryBuilder tqb = QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, idString );
final DeleteByQueryResponse response = esProvider.getClient()
.prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute().actionGet();
@@ -526,11 +524,14 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
- public void deletePreviousVersions( final Id id, final UUID version ) {
+ public void deletePreviousVersions( final Id entityId, final UUID version ) {
+
+ String idString = IndexingUtils.idString( entityId ).toLowerCase();
final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
- QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, id.getUuid().toString().toLowerCase()),
- FilterBuilders.rangeFilter( ENTITY_VERSION_FIELDNAME ).lt( version.timestamp() ) );
+ QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, idString ),
+ FilterBuilders.rangeFilter( ENTITY_VERSION_FIELDNAME ).lt( version.timestamp() )
+ );
final DeleteByQueryResponse response = esProvider.getClient()
.prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( fqb ).execute().actionGet();
@@ -538,14 +539,14 @@ public class EsEntityIndexImpl implements EntityIndex {
//error message needs to be retooled so that it describes the entity more throughly
logger.debug( "Deleted entity {}:{} with version {} from all "
+ "index scopes with response status = {}",
- new Object[] { id.getType(), id.getUuid(), version, response.status().toString() } );
+ new Object[] { entityId.getType(), entityId.getUuid(), version, response.status().toString() } );
checkDeleteByQueryResponse( fqb, response );
}
/**
- * Validate the response doens't contain errors, if it does, fail fast at the first error we encounter
+ * Validate the response doesn't contain errors, if it does, fail fast at the first error we encounter
*/
private void checkDeleteByQueryResponse(
final QueryBuilder query, final DeleteByQueryResponse response ) {
[39/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/88d49cb8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/88d49cb8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/88d49cb8
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 88d49cb8d7123276db0a4dabb78061f4465b0ccf
Parents: 5f0dc34 ce20abe
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Nov 25 11:24:00 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Nov 25 11:24:00 2014 -0500
----------------------------------------------------------------------
.../org/apache/usergrid/CoreApplication.java | 17 +-
.../migration/EntityDataMigrationIT.java | 16 +-
.../migration/EntityTypeMappingMigrationIT.java | 17 +
.../migration/GraphShardVersionMigrationIT.java | 20 +-
.../migration/MigrationTestRule.java | 101 ++++
.../org/apache/usergrid/persistence/GeoIT.java | 579 ++++++++++---------
.../applications/queries/BasicGeoTests.java | 268 +++++++++
.../applications/queries/basicGeoTests.java | 184 ------
8 files changed, 728 insertions(+), 474 deletions(-)
----------------------------------------------------------------------
[49/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/37940e34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/37940e34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/37940e34
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 37940e340dbf1e75207913c9cb64530e4578e2f5
Parents: a8cc587 c6254f2
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Dec 15 11:31:11 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Dec 15 11:31:11 2014 -0700
----------------------------------------------------------------------
.../apache/usergrid/services/notifications/apns/APNsAdapter.java | 4 ++--
.../usergrid/services/notifications/apns/EntityPushManager.java | 3 ++-
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[02/50] incubator-usergrid git commit: Created extra step to help
mock out the test. Created new test to check to see how one listener is
handled
Posted by sf...@apache.org.
Created extra step to help mock out the test.
Created new test to check to see how one listener is handled
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e6760f25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e6760f25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e6760f25
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: e6760f2569a9b1fb10ba5155120d59ecf5f4b733
Parents: 67f7422
Author: grey <gr...@apigee.com>
Authored: Wed Oct 29 16:25:41 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Wed Oct 29 16:25:41 2014 -0700
----------------------------------------------------------------------
.../persistence/collection/impl/EntityVersionCreatedTask.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e6760f25/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
index b78b09c..cef4248 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTask.java
@@ -122,7 +122,9 @@ public class EntityVersionCreatedTask implements Task<Void> {
}
if ( listenerSize == 1 ) {
- listeners.iterator().next().versionCreated( collectionScope,entity );
+ EntityVersionCreated entityVersionCreated = listeners.iterator().next();
+
+ entityVersionCreated.versionCreated( collectionScope,entity );
return;
}
[28/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1f71cf6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1f71cf6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1f71cf6c
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 1f71cf6c7337905608b3994a36870191cf98446e
Parents: 8457acf 87d6e43
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Nov 10 09:22:41 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Nov 10 09:22:41 2014 -0500
----------------------------------------------------------------------
.../index/impl/ElasticSearchResource.java | 30 +++-
stack/loadtests/README.md | 4 +-
.../org/apache/usergrid/helpers/Setup.scala | 46 ++++---
.../scenarios/ConnectionScenarios.scala | 10 +-
.../scenarios/NotificationScenarios.scala | 6 +-
.../org/apache/usergrid/settings/Settings.scala | 23 ++--
.../usergrid/simulations/AppSimulation.scala | 16 +--
.../usergrid/cassandra/CassandraResource.java | 138 +++++++++++++++----
8 files changed, 192 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
[23/50] incubator-usergrid git commit: Removed task executor from
tests to separate concerns.
Posted by sf...@apache.org.
Removed task executor from tests to separate concerns.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3294f009
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3294f009
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3294f009
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 3294f00928c5b066caa5bd84dd1aec2d563c3468
Parents: e759a79
Author: grey <gr...@apigee.com>
Authored: Tue Nov 4 15:35:25 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Nov 4 15:35:25 2014 -0800
----------------------------------------------------------------------
.../impl/EntityVersionCleanupTaskTest.java | 51 ++++++++++----------
.../impl/EntityVersionCreatedTaskTest.java | 34 +++++++------
2 files changed, 45 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3294f009/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
index 0e86027..1c7d510 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.junit.AfterClass;
import org.junit.Test;
+import org.junit.Assert;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
@@ -156,11 +157,11 @@ public class EntityVersionCleanupTaskTest {
when( ess.load( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
.thenReturn(mel.iterator() );
- // start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
-
- // wait for the task
- future.get();
+ try {
+ cleanupTask.call();
+ }catch(Exception e){
+ Assert.fail( e.getMessage() );
+ }
// verify it was run
verify( entityBatch ).execute();
@@ -257,10 +258,11 @@ public class EntityVersionCleanupTaskTest {
.thenReturn(mel.iterator() );
//start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
-
- //wait for the task
- future.get();
+ try {
+ cleanupTask.call();
+ }catch(Exception e){
+ Assert.fail( e.getMessage() );
+ }
// These last two verify statements do not make sense. We cannot assert that the entity
@@ -368,11 +370,11 @@ public class EntityVersionCleanupTaskTest {
.thenReturn(mel.iterator() );
- //start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
-
- //wait for the task
- future.get();
+ try {
+ cleanupTask.call();
+ }catch(Exception e){
+ Assert.fail( e.getMessage() );
+ }
//we deleted the version
//verify it was run
@@ -480,12 +482,11 @@ public class EntityVersionCleanupTaskTest {
when( ess.load( same( appScope ), same( entityId ), any(UUID.class), any(Integer.class) ) )
.thenReturn(mel.iterator() );
- //start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
-
- //wait for the task
- future.get();
-
+ try {
+ cleanupTask.call();
+ }catch(Exception e){
+ Assert.fail( e.getMessage() );
+ }
//we deleted the version
//verify we deleted everything
verify( entityBatch, times( 1 ) ).mergeShallow( any( MutationBatch.class ) );
@@ -740,11 +741,11 @@ public class EntityVersionCleanupTaskTest {
.thenReturn(mel.iterator() );
- //start the task
- ListenableFuture<Void> future = taskExecutor.submit( cleanupTask );
-
- //wait for the task
- future.get();
+ try {
+ cleanupTask.call();
+ }catch(Exception e){
+ Assert.fail(e.getMessage());
+ }
//we deleted the version
//verify it was run
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3294f009/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index 24ea280..3ba8fe1 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -50,6 +50,7 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Test;
import static org.mockito.Matchers.any;
@@ -98,10 +99,15 @@ public class EntityVersionCreatedTaskTest {
EntityVersionCreatedTask entityVersionCreatedTask =
new EntityVersionCreatedTask( appScope, listeners, entity);
- ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
+ try {
+ entityVersionCreatedTask.call();
+ }catch(Exception e){
+ Assert.fail(e.getMessage());
+ }
+
// wait for the task
- future.get();
+ // future.get();
//mocked listener makes sure that the task is called
verify( listeners ).size();
@@ -139,11 +145,12 @@ public class EntityVersionCreatedTaskTest {
EntityVersionCreatedTask entityVersionCreatedTask =
new EntityVersionCreatedTask( appScope, listeners, entity);
- ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
-
- // wait for the task
- future.get();
+ try {
+ entityVersionCreatedTask.call();
+ }catch(Exception e){
+ Assert.fail(e.getMessage());
+ }
//mocked listener makes sure that the task is called
verify( listeners ).size();
verify( listeners ).iterator();
@@ -184,15 +191,15 @@ public class EntityVersionCreatedTaskTest {
when ( helper.next() ).thenReturn( listener1,listener2,listener3);
- ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
-
- //wait for the task
- //intentionally fails due to difficulty mocking observable
try {
- future.get();
+ entityVersionCreatedTask.call();
}catch(Exception e){
;
}
+ //ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
+
+ //wait for the task
+ //intentionally fails due to difficulty mocking observable
//mocked listener makes sure that the task is called
verify( listeners ).size();
@@ -234,10 +241,7 @@ public class EntityVersionCreatedTaskTest {
EntityVersionCreatedTask entityVersionCreatedTask =
new EntityVersionCreatedTask( appScope, listeners, entity);
- ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
-
- // wait for the task
- future.get();
+ entityVersionCreatedTask.rejected();
//mocked listener makes sure that the task is called
verify( listeners ).size();
[44/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bf47eb37
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bf47eb37
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bf47eb37
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: bf47eb371a753c9261d82c75390482141c551627
Parents: bd3b422
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Dec 8 15:53:47 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Dec 8 15:53:47 2014 -0500
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 9 ---------
1 file changed, 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf47eb37/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 0614220..51ee915 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -254,7 +254,6 @@ public class EsEntityIndexImpl implements EntityIndex {
logger.info( "Successfully deleted all documents in index {} and type {}",
alias, VERIFY_TYPE );
-
return true;
}
};
@@ -360,10 +359,8 @@ public class EsEntityIndexImpl implements EntityIndex {
logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ",
new Object[] { this.alias, context, entityTypes, srb
} );
-
}
-
try {
searchResponse = srb.execute().actionGet();
}
@@ -488,17 +485,11 @@ public class EsEntityIndexImpl implements EntityIndex {
final QueryBuilder queryBuilder =
QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
-
final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
.setTypes(searchTypes.getTypeNames())
.setScroll(cursorTimeout + "m")
.setQuery(queryBuilder);
-
- final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
- .setTypes(searchTypes.getTypeNames())
- .setScroll(cursorTimeout + "m").setQuery(queryBuilder);
-
final SearchResponse searchResponse;
try {
searchResponse = srb.execute().actionGet();
[17/50] incubator-usergrid git commit: Added an extra test to check
that multiple listeners function correctly.
Posted by sf...@apache.org.
Added an extra test to check that multiple listeners function correctly.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e9f876c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e9f876c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e9f876c5
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: e9f876c5c1b336cdffff85c06457bc926dfe479d
Parents: 54a64e6
Author: grey <gr...@apigee.com>
Authored: Mon Nov 3 15:44:56 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Mon Nov 3 15:44:56 2014 -0800
----------------------------------------------------------------------
.../impl/EntityVersionCreatedTaskTest.java | 72 ++++++++++++++++++++
1 file changed, 72 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e9f876c5/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
index b25b5b4..9d72665 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCreatedTaskTest.java
@@ -17,22 +17,45 @@
*/
package org.apache.usergrid.persistence.collection.impl;
+import com.google.common.base.Optional;
import com.google.common.util.concurrent.ListenableFuture;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
+import java.util.ArrayList;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
+import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.util.LogEntryMock;
import org.apache.usergrid.persistence.core.task.NamedTaskExecutorImpl;
import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
import org.junit.AfterClass;
import org.junit.Test;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.same;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -128,6 +151,55 @@ public class EntityVersionCreatedTaskTest {
}
+ @Test(timeout=10000)
+ public void multipleListener()
+ throws ExecutionException, InterruptedException, ConnectionException {
+
+ final int sizeToReturn = 3;
+
+ final Set<EntityVersionCreated> listeners = mock( Set.class );
+ final Iterator<EntityVersionCreated> helper = mock(Iterator.class);
+
+ when ( listeners.size()).thenReturn( 3 );
+ when ( listeners.iterator()).thenReturn( helper );
+
+ final Id applicationId = new SimpleId( "application" );
+
+ final CollectionScope appScope = new CollectionScopeImpl(
+ applicationId, applicationId, "users" );
+
+ final Id entityId = new SimpleId( "user" );
+ final Entity entity = new Entity( entityId );
+
+ // start the task
+
+ EntityVersionCreatedTask entityVersionCreatedTask =
+ new EntityVersionCreatedTask( appScope, listeners, entity);
+
+ final CountDownLatch latch = new CountDownLatch( sizeToReturn );
+
+ final EntityVersionCreatedTest listener1 = new EntityVersionCreatedTest(latch);
+ final EntityVersionCreatedTest listener2 = new EntityVersionCreatedTest(latch);
+ final EntityVersionCreatedTest listener3 = new EntityVersionCreatedTest(latch);
+
+ when ( helper.next() ).thenReturn( listener1,listener2,listener3);
+
+ ListenableFuture<Void> future = taskExecutor.submit( entityVersionCreatedTask );
+
+ //wait for the task
+ //intentionally fails due to difficulty mocking observable
+ try {
+ future.get();
+ }catch(Exception e){
+ ;
+ }
+
+ //mocked listener makes sure that the task is called
+ verify( listeners ).size();
+ //verifies that the observable made listener iterate.
+ verify( listeners ).iterator();
+ }
+
private static class EntityVersionCreatedTest implements EntityVersionCreated {
final CountDownLatch invocationLatch;
[42/50] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-events
Conflicts:
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ed0ea46d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ed0ea46d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ed0ea46d
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: ed0ea46dacdb49dfce7219ea544c6bfd2cd1cdfa
Parents: 532a02c 7645a79
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Dec 4 17:23:31 2014 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Dec 4 17:23:31 2014 -0500
----------------------------------------------------------------------
portal/Gruntfile.js | 193 ++++++++-----
portal/build.sh | 2 -
portal/config.js | 94 +++++--
.../src/main/groovy/NodeRegistry.groovy | 12 +-
.../corepersistence/CpEntityManagerFactory.java | 5 +
.../HybridEntityManagerFactory.java | 5 +
.../events/EntityVersionDeletedHandler.java | 10 +-
.../persistence/EntityManagerFactory.java | 10 +
.../cassandra/EntityManagerFactoryImpl.java | 5 +
.../collection/event/EntityVersionDeleted.java | 1 -
.../usergrid/persistence/index/EntityIndex.java | 16 ++
.../usergrid/persistence/index/IndexFig.java | 6 +
.../persistence/index/IndexIdentifier.java | 88 ++++++
.../index/impl/EsEntityIndexBatchImpl.java | 15 +-
.../index/impl/EsEntityIndexImpl.java | 280 +++++++++++--------
.../persistence/index/impl/IndexingUtils.java | 20 +-
.../persistence/index/impl/EntityIndexTest.java | 75 ++++-
.../simulations/PostUsersSimulation.scala | 2 +-
.../org/apache/usergrid/rest/IndexResource.java | 227 +++++++++++++++
.../apache/usergrid/rest/SystemResource.java | 151 +---------
.../apache/usergrid/rest/AbstractRestIT.java | 2 +-
.../apache/usergrid/rest/IndexResourceIT.java | 80 ++++++
22 files changed, 896 insertions(+), 403 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
index a6e971a,0000000..23097c4
mode 100644,000000..100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedHandler.java
@@@ -1,84 -1,0 +1,84 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+package org.apache.usergrid.corepersistence.events;
+
++import com.google.inject.Inject;
++import java.util.List;
++import java.util.UUID;
+import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
++import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import rx.functions.Func1;
+import rx.schedulers.Schedulers;
+
- import java.util.List;
-
- import com.google.inject.Inject;
- import org.apache.usergrid.persistence.EntityManagerFactory;
-
+
+/**
+ * Remove Entity index when specific version of Entity is deleted.
+ * TODO: do we need this? Don't our version-created and entity-deleted handlers take care of this?
+ * If we do need it then it should be wired in via GuiceModule in the corepersistence package.
+ */
+public class EntityVersionDeletedHandler implements EntityVersionDeleted {
+
+ private final SerializationFig serializationFig;
+
+ private EntityManagerFactory emf;
+
+ @Inject
+ public EntityVersionDeletedHandler(SerializationFig fig, EntityManagerFactory emf) {
+ this.serializationFig = fig;
+ }
+
+ @Override
+ public void versionDeleted(
+ final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
+
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)emf;
+
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+
+ final EntityIndexBatch eibatch = ei.createBatch();
+
+ final IndexScope indexScope = new IndexScopeImpl(
+ new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
+ scope.getName()
+ );
+ rx.Observable.from(entityVersions)
+ .subscribeOn(Schedulers.io())
+ .buffer(serializationFig.getBufferSize())
+ .map(new Func1<List<MvccEntity>, List<MvccEntity>>() {
+ @Override
+ public List<MvccEntity> call(List<MvccEntity> entityList) {
+ for (MvccEntity entity : entityList) {
+ eibatch.deindex(indexScope, entityId, entity.getVersion());
+ }
+ eibatch.execute();
+ return entityList;
+ }
+ }).toBlocking().last();
+ }
++
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
index 65a34e0,aedcf3a..c76be8e
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
@@@ -32,7 -33,7 +32,6 @@@ import org.apache.usergrid.persistence.
*/
public interface EntityVersionDeleted {
--
/**
* The version specified was removed.
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index b099678,a3ca8a5..2f555ed
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@@ -64,9 -67,8 +65,9 @@@ import static org.apache.usergrid.persi
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
+ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@@ -177,10 -180,9 +179,9 @@@
}
- log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
+ log.debug( "De-indexing type {} with documentId '{}'", entityType, indexId );
- bulkRequest.add( client.prepareDelete(
- indexName, entityType, indexId ).setRefresh( refresh ) );
+ bulkRequest.add( client.prepareDelete(alias.getWriteAlias(), entityType, indexId ).setRefresh( refresh ) );
log.debug( "Deindexed Entity with index id " + indexId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index aa797e2,d546092..86826a5
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -18,13 -18,15 +18,38 @@@
package org.apache.usergrid.persistence.index.impl;
++import com.google.common.collect.ImmutableMap;
++import com.google.inject.Inject;
++import com.google.inject.assistedinject.Assisted;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
--
-import org.apache.usergrid.persistence.index.*;
-import org.apache.usergrid.persistence.index.utils.StringUtils;
++import org.apache.commons.lang.StringUtils;
++import org.apache.usergrid.persistence.core.scope.ApplicationScope;
++import org.apache.usergrid.persistence.core.util.Health;
++import org.apache.usergrid.persistence.core.util.ValidationUtils;
++import org.apache.usergrid.persistence.index.EntityIndex;
++import org.apache.usergrid.persistence.index.EntityIndexBatch;
++import org.apache.usergrid.persistence.index.IndexFig;
++import org.apache.usergrid.persistence.index.IndexIdentifier;
++import org.apache.usergrid.persistence.index.IndexScope;
++import org.apache.usergrid.persistence.index.SearchTypes;
++import org.apache.usergrid.persistence.index.exceptions.IndexException;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_VERSION_FIELDNAME;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
++import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
++import org.apache.usergrid.persistence.index.query.CandidateResult;
++import org.apache.usergrid.persistence.index.query.CandidateResults;
++import org.apache.usergrid.persistence.index.query.Query;
++import org.apache.usergrid.persistence.model.entity.Id;
++import org.apache.usergrid.persistence.model.entity.SimpleId;
++import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
@@@ -59,32 -59,26 +87,6 @@@ import org.elasticsearch.search.sort.So
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
--import org.apache.usergrid.persistence.core.scope.ApplicationScope;
--import org.apache.usergrid.persistence.core.util.Health;
--import org.apache.usergrid.persistence.core.util.ValidationUtils;
- import org.apache.usergrid.persistence.index.EntityIndex;
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
- import org.apache.usergrid.persistence.index.IndexFig;
- import org.apache.usergrid.persistence.index.IndexScope;
- import org.apache.usergrid.persistence.index.SearchTypes;
--import org.apache.usergrid.persistence.index.exceptions.IndexException;
--import org.apache.usergrid.persistence.index.query.CandidateResult;
--import org.apache.usergrid.persistence.index.query.CandidateResults;
--import org.apache.usergrid.persistence.index.query.Query;
--import org.apache.usergrid.persistence.model.entity.Id;
--import org.apache.usergrid.persistence.model.entity.SimpleId;
--import org.apache.usergrid.persistence.model.util.UUIDGenerator;
--
--import com.google.common.collect.ImmutableMap;
--import com.google.inject.Inject;
--import com.google.inject.assistedinject.Assisted;
--
--import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
- import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
--import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
--import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
--import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
--
/**
* Implements index using ElasticSearch Java API.
@@@ -125,11 -120,8 +128,10 @@@ public class EsEntityIndexImpl implemen
@Inject
- public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
- final EsProvider provider ) {
- public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config, final EsProvider provider ) {
++ public EsEntityIndexImpl(
++ @Assisted final ApplicationScope appScope, final IndexFig config, final EsProvider provider ) {
+
ValidationUtils.validateApplicationScope( appScope );
-
this.applicationScope = appScope;
this.esProvider = provider;
this.config = config;
@@@ -147,34 -147,54 +157,63 @@@
createMappings();
}
- final AdminClient admin = esProvider.getClient().admin();
+ //get index name with suffix attached
+ String indexName = indexIdentifier.getIndex(normalizedSuffix);
+
+ //Create index
+ try {
+ final AdminClient admin = esProvider.getClient().admin();
+ Settings settings = ImmutableSettings.settingsBuilder().put("index.number_of_shards", numberOfShards)
+ .put("index.number_of_replicas", numberOfReplicas).build();
+ final CreateIndexResponse cir = admin.indices().prepareCreate(indexName).setSettings(settings).execute().actionGet();
+ logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
+ } catch (IndexAlreadyExistsException e) {
+ logger.info("Index Name [{}] already exists", indexName);
+ }
+
+ addAlias(normalizedSuffix);
- final int numberOfShards = config.getNumberOfShards();
- final int numberOfReplicas = config.getNumberOfReplicas();
+ testNewIndex();
+ } catch (IndexAlreadyExistsException expected) {
+ // this is expected to happen if index already exists, it's a no-op and swallow
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to initialize index", e);
+ }
+ }
- Settings settings = ImmutableSettings.settingsBuilder()
- .put( "index.number_of_shards", numberOfShards )
- .put( "index.number_of_replicas", numberOfReplicas ).build();
+ @Override
+ public void addAlias(final String indexSuffix) {
+ try {
+ Boolean isAck;
+ String indexName = indexIdentifier.getIndex(indexSuffix);
+ final AdminClient adminClient = esProvider.getClient().admin();
+
- final CreateIndexResponse cir = admin.indices().prepareCreate( indexName )
- .setSettings( settings ).execute().actionGet();
+ //remove write alias, can only have one
- ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices().getAliases(new GetAliasesRequest(alias.getWriteAlias())).actionGet().getAliases();
++ ImmutableOpenMap<String,List<AliasMetaData>> aliasMap = adminClient.indices()
++ .getAliases(new GetAliasesRequest(alias.getWriteAlias())).actionGet().getAliases();
+
- logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
+ String[] indexNames = aliasMap.keys().toArray(String.class);
+ for(String currentIndex : indexNames){
- isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
- logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",currentIndex, alias, isAck);
++ isAck = adminClient.indices().prepareAliases().removeAlias(
++ currentIndex,alias.getWriteAlias()).execute().actionGet().isAcknowledged();
++ logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]",
++ currentIndex, alias, isAck);
- // create the document, this ensures the index is ready
+ }
+
- // Immediately create a document and remove it to ensure the entire cluster is ready
- // to receive documents. Occasionally we see errors.
- // See this post: http://s.apache.org/index-missing-exception
+ //add read alias
- isAck = adminClient.indices().prepareAliases().addAlias(indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
++ isAck = adminClient.indices().prepareAliases().addAlias(
++ indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
+ logger.info("Created new read Alias Name [{}] ACK=[{}]", alias, isAck);
+
- testNewIndex();
- }
- catch ( IndexAlreadyExistsException expected ) {
- // this is expected to happen if index already exists, it's a no-op and swallow
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Unable to initialize index", e );
+ //add write alias
- isAck = adminClient.indices().prepareAliases().addAlias(indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
++ isAck = adminClient.indices().prepareAliases().addAlias(
++ indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
+ logger.info("Created new write Alias Name [{}] ACK=[{}]", alias, isAck);
+
+ } catch (Exception e) {
+ logger.warn("Failed to create alias ", e);
}
}
@@@ -193,19 -217,18 +236,20 @@@
public boolean doOp() {
final String tempId = UUIDGenerator.newTimeUUID().toString();
- esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId )
- .setSource( DEFAULT_PAYLOAD ).get();
- esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
- .get();
++ esProvider.getClient().prepareIndex( alias.getWriteAlias(), VERIFY_TYPE, tempId )
++ .setSource( DEFAULT_PAYLOAD ).get();
- logger.info( "Successfully created new document with docId {} in index {} type {}",
- tempId, indexName, VERIFY_TYPE );
- logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
- alias, VERIFY_TYPE );
++ logger.info( "Created new document with docId {} in index {} and type {}",
++ tempId, alias, VERIFY_TYPE );
// delete all types, this way if we miss one it will get cleaned up
- esProvider.getClient().prepareDeleteByQuery( indexName )
- .setTypes( VERIFY_TYPE )
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
- esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() ).setTypes(VERIFY_TYPE)
- .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
++ esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() )
++ .setTypes(VERIFY_TYPE)
++ .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+
- logger.info( "Successfully deleted all documents in index {} and type {}", alias, VERIFY_TYPE );
++ logger.info( "Deleted all documents in index {} and type {}",
++ alias, VERIFY_TYPE );
- logger.info( "Successfully deleted all documents in index {} and type {}",
- indexName, VERIFY_TYPE );
return true;
}
@@@ -221,16 -244,16 +265,16 @@@
*/
private void createMappings() throws IOException {
- XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
- XContentFactory.jsonBuilder(), "_default_" );
+ XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
+ XContentFactory.jsonBuilder(), "_default_");
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
- .preparePutTemplate( "usergrid_template" )
+ .preparePutTemplate("usergrid_template")
// set mapping as the default for all types
- .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_", xcb )
+ .setTemplate(config.getIndexPrefix() + "*").addMapping( "_default_", xcb )
.execute().actionGet();
- if(!pitr.isAcknowledged()){
+ if ( !pitr.isAcknowledged() ) {
throw new IndexException( "Unable to create default mappings" );
}
}
@@@ -256,9 -279,10 +300,9 @@@
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName )
- .setTypes( entityTypes )
- .setScroll( cursorTimeout + "m" ).setQuery( qb );
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes(entityTypes)
- .setScroll(cursorTimeout + "m").setQuery(qb);
-
+
++ SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
++ .setTypes(entityTypes).setScroll(cursorTimeout + "m").setQuery(qb);
final FilterBuilder fb = query.createFilterBuilder();
@@@ -307,9 -331,12 +351,11 @@@
}
-
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ",
- new Object[] { this.indexName, context, entityTypes, srb } );
+ logger.debug( "Searching index {}\n scope{} \n type {}\n query {} ", new Object[] {
+ this.alias, context, entityTypes, srb
+ } );
+
}
@@@ -397,9 -424,9 +443,10 @@@
@Override
public boolean doOp() {
try {
- esProvider.getClient().admin().indices().prepareRefresh( indexName )
- .execute().actionGet();
- logger.debug( "Refreshed index: " + indexName );
- esProvider.getClient().admin().indices().prepareRefresh( alias.getWriteAlias() ).execute().actionGet();
++ esProvider.getClient().admin().indices()
++ .prepareRefresh( alias.getWriteAlias() ).execute().actionGet();
+ logger.debug( "Refreshed index: " + alias);
+
return true;
}
catch ( IndexMissingException e ) {
@@@ -430,15 -457,17 +477,15 @@@
//since we don't have paging inputs, there's no point in executing a query for paging.
- final String context = IndexingUtils.createContextName( scope );
+ final String context = IndexingUtils.createContextName(scope);
final SearchTypes searchTypes = SearchTypes.fromTypes( id.getType() );
- final QueryBuilder queryBuilder = QueryBuilders.termQuery(
- IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
- final QueryBuilder queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
-
-
- final SearchRequestBuilder srb =
- esProvider.getClient().prepareSearch( alias.getReadAlias() ).setTypes(searchTypes.getTypeNames())
- .setScroll(cursorTimeout + "m").setQuery(queryBuilder);
-
++ final QueryBuilder queryBuilder =
++ QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
- final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName )
- .setTypes( searchTypes.getTypeNames() ).setScroll( cursorTimeout + "m" )
- .setQuery( queryBuilder );
++ final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
++ .setTypes(searchTypes.getTypeNames())
++ .setScroll(cursorTimeout + "m").setQuery(queryBuilder);
final SearchResponse searchResponse;
try {
@@@ -457,73 -486,17 +504,80 @@@
}
+ @Override
+ public void deleteAllVersionsOfEntity( Id entityId ) {
+
- final TermQueryBuilder tqb =
- QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, entityId.getUuid().toString().toLowerCase() );
++ final TermQueryBuilder tqb = QueryBuilders.termQuery(
++ ENTITYID_ID_FIELDNAME, entityId.getUuid().toString().toLowerCase() );
+
- final DeleteByQueryResponse response =
- esProvider.getClient().prepareDeleteByQuery( indexName ).setQuery( tqb ).execute().actionGet();
++ final DeleteByQueryResponse response = esProvider.getClient()
++ .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute().actionGet();
+
+
+ logger.debug( "Deleted entity {}:{} from all index scopes with response status = {}",
- new Object[] { entityId.getType(), entityId.getUuid(), response.status().toString() } );
++ new Object[] { entityId.getType(), entityId.getUuid(), response.status().toString() });
+
+ checkDeleteByQueryResponse( tqb, response );
+
+ }
+
+
+ @Override
+ public void deletePreviousVersions( final Id id, final UUID version ) {
+
+ final FilteredQueryBuilder fqb = QueryBuilders.filteredQuery(
- QueryBuilders.termQuery( IndexingUtils.ENTITYID_ID_FIELDNAME, id.getUuid().toString().toLowerCase() ),
- FilterBuilders.rangeFilter( IndexingUtils.ENTITY_VERSION_FIELDNAME ).lt( version.timestamp() ) );
++ QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, id.getUuid().toString().toLowerCase()),
++ FilterBuilders.rangeFilter( ENTITY_VERSION_FIELDNAME ).lt( version.timestamp() ) );
+
- final DeleteByQueryResponse response =
- esProvider.getClient().prepareDeleteByQuery( indexName ).setQuery( fqb ).execute().actionGet();
++ final DeleteByQueryResponse response = esProvider.getClient()
++ .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( fqb ).execute().actionGet();
+
+ //error message needs to be retooled so that it describes the entity more throughly
- logger.debug( "Deleted entity {}:{} with version {} from all index scopes with response status = {}",
- new Object[] { id.getType(), id.getUuid(), version, response.status().toString() } );
++ logger.debug( "Deleted entity {}:{} with version {} from all "
++ + "index scopes with response status = {}",
++ new Object[] { id.getType(), id.getUuid(), version, response.status().toString() } );
+
+ checkDeleteByQueryResponse( fqb, response );
+ }
+
+
+ /**
+ * Validate the response doens't contain errors, if it does, fail fast at the first error we encounter
- * @param query
- * @param response
+ */
- private void checkDeleteByQueryResponse( final QueryBuilder query, final DeleteByQueryResponse response ) {
++ private void checkDeleteByQueryResponse(
++ final QueryBuilder query, final DeleteByQueryResponse response ) {
++
+ for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) {
+ final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures();
+
+ for ( ShardOperationFailedException failedException : failures ) {
- throw new IndexException( String.format(
- "Unable to delete by query %s. Failed with code %d and reason %s on shard %s in index %s",
- query.toString(), failedException.status(), failedException.reason(), failedException.shardId(),
++ throw new IndexException( String.format("Unable to delete by query %s. "
++ + "Failed with code %d and reason %s on shard %s in index %s",
++ query.toString(),
++ failedException.status(),
++ failedException.reason(),
++ failedException.shardId(),
+ failedException.index() ) );
+ }
+
+ }
+ }
+
+
/**
* For testing only.
*/
public void deleteIndex() {
AdminClient adminClient = esProvider.getClient().admin();
- DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
- DeleteIndexResponse response = adminClient.indices().prepareDelete( indexIdentifier.getIndex(null) ).get();
++
++ DeleteIndexResponse response = adminClient.indices()
++ .prepareDelete( indexIdentifier.getIndex(null) ).get();
++
if ( response.isAcknowledged() ) {
- logger.info( "Deleted index: " + indexName );
+ logger.info( "Deleted index: " + alias);
}
else {
- logger.info( "Failed to delete index " + indexName );
+ logger.info( "Failed to delete index " + alias);
}
}
@@@ -582,7 -555,8 +636,7 @@@
try {
ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
- .health( new ClusterHealthRequest( new String[] { indexName } ) ).get();
- .health(new ClusterHealthRequest(new String[]{indexIdentifier.getIndex(null)}))
- .get();
++ .health(new ClusterHealthRequest(new String[]{indexIdentifier.getIndex(null)})).get();
return Health.valueOf( chr.getStatus().name() );
}
catch ( Exception ex ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ed0ea46d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
[13/50] incubator-usergrid git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-events
Posted by sf...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-events
Conflicts:
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4a36aa16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4a36aa16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4a36aa16
Branch: refs/heads/UG-rest-test-framework-overhaul
Commit: 4a36aa16ccdb7b60122bcf32522788706beff806
Parents: 7000bd9 b6a8bc6
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Oct 31 14:09:36 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Oct 31 14:09:36 2014 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 2 +
.../corepersistence/CpEntityManagerFactory.java | 20 ++++
.../HybridEntityManagerFactory.java | 10 ++
.../persistence/EntityManagerFactory.java | 4 +
.../cassandra/EntityManagerFactoryImpl.java | 10 ++
.../collection/EntityCollectionManager.java | 9 +-
.../impl/EntityCollectionManagerImpl.java | 110 ++++++++++++-------
.../serialization/SerializationFig.java | 4 -
.../collection/EntityCollectionManagerIT.java | 12 ++
.../usergrid/persistence/index/EntityIndex.java | 11 +-
.../usergrid/persistence/index/IndexFig.java | 12 ++
.../index/impl/EsEntityIndexImpl.java | 76 ++++++++-----
.../persistence/index/impl/EsProvider.java | 2 +
.../impl/EntityConnectionIndexImplTest.java | 22 +++-
stack/loadtests/src/main/scripts/gatling-mvn.sh | 3 +
.../src/test/resources/logback-test.xml | 41 +++++++
stack/loadtests/src/test/resources/logback.xml | 3 +-
.../apache/usergrid/rest/SwaggerServlet.java | 5 +-
.../rest/filters/HealthCheckFilter.java | 68 ++++++++++++
stack/rest/src/main/webapp/WEB-INF/web.xml | 10 ++
20 files changed, 351 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4a36aa16/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4a36aa16/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 5b75f22,360d8e0..0205824
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@@ -70,10 -68,21 +67,16 @@@ public interface EntityCollectionManage
*/
public Observable<EntitySet> load(Collection<Id> entityIds);
-
/**
* Takes the change and reloads an entity with all changes applied in this entity applied.
- * The resulting entity from calling load will be the previous version of this entity + the entity
- * in this object applied to it.
- * @param entity
- * @return
+ * The resulting entity from calling load will be the previous version of this entity plus
+ * the entity in this object applied to it.
*/
public Observable<Entity> update ( Entity entity );
+
+ /**
+ * Return true if connection query index system (Cassandra) is healthy.
+ */
+ public boolean isHealthy();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4a36aa16/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 5b6265c,f24334e..26ab6aa
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -34,7 -31,10 +34,6 @@@ import org.apache.usergrid.persistence.
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.collection.guice.Write;
import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
@@@ -58,6 -57,12 +55,14 @@@ import org.apache.usergrid.persistence.
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
+ import com.netflix.astyanax.Keyspace;
+ import com.netflix.astyanax.connectionpool.OperationResult;
+ import com.netflix.astyanax.model.ColumnFamily;
+ import com.netflix.astyanax.model.CqlResult;
+ import com.netflix.astyanax.serializers.StringSerializer;
++import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
+ import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
++import org.apache.usergrid.persistence.core.task.TaskExecutor;
import rx.Observable;
import rx.Subscriber;
@@@ -73,11 -78,11 +78,10 @@@ import rx.schedulers.Schedulers
*/
public class EntityCollectionManagerImpl implements EntityCollectionManager {
- private static final Logger log = LoggerFactory.getLogger(EntityCollectionManagerImpl.class);
+ private static final Logger logger = LoggerFactory.getLogger(EntityCollectionManagerImpl.class);
private final CollectionScope collectionScope;
- private final UUIDService uuidService;
-
//start stages
private final WriteStart writeStart;
private final WriteStart writeUpdate;
@@@ -90,37 -96,28 +94,37 @@@
private final MarkStart markStart;
private final MarkCommit markCommit;
- private final TaskExecutor taskExecutor;
-
- private EntityVersionCleanupFactory entityVersionCleanupFactory;
- private EntityVersionCreatedFactory entityVersionCreatedFactory;
-
private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
private final MvccEntitySerializationStrategy entitySerializationStrategy;
- private final EntityDeletedFactory entityDeletedFactory;
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
++ private final EntityVersionCleanupFactory entityVersionCleanupFactory;
++ private final EntityVersionCreatedFactory entityVersionCreatedFactory;
++ private final EntityDeletedFactory entityDeletedFactory;
++ private final TaskExecutor taskExecutor;
++
+ private final Keyspace keyspace;
+ private SerializationFig config;
+
@Inject
- public EntityCollectionManagerImpl(
- final UUIDService uuidService,
- @Write final WriteStart writeStart,
- @WriteUpdate final WriteStart writeUpdate,
- final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit,
- final RollbackAction rollback,
- final MarkStart markStart,
- final MarkCommit markCommit,
- final EntityVersionCleanupFactory entityVersionCleanupFactory,
- final EntityVersionCreatedFactory entityVersionCreatedFactory,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final EntityDeletedFactory entityDeletedFactory,
+ public EntityCollectionManagerImpl(
+ @Write final WriteStart writeStart,
+ @WriteUpdate final WriteStart writeUpdate,
+ final WriteUniqueVerify writeVerifyUnique,
+ final WriteOptimisticVerify writeOptimisticVerify,
+ final WriteCommit writeCommit, final RollbackAction rollback,
+ final MarkStart markStart, final MarkCommit markCommit,
+ final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
+ final Keyspace keyspace,
+ final SerializationFig config,
++ final EntityVersionCleanupFactory entityVersionCleanupFactory,
++ final EntityVersionCreatedFactory entityVersionCreatedFactory,
++ final EntityDeletedFactory entityDeletedFactory,
+ @CollectionTaskExecutor final TaskExecutor taskExecutor,
- @Assisted final CollectionScope collectionScope
-
+ @Assisted final CollectionScope collectionScope
) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
@@@ -141,10 -134,11 +141,15 @@@
this.markStart = markStart;
this.markCommit = markCommit;
- this.uuidService = uuidService;
- this.collectionScope = collectionScope;
- this.taskExecutor = taskExecutor;
+ this.keyspace = keyspace;
+ this.config = config;
+
+ this.entityVersionCleanupFactory = entityVersionCleanupFactory;
++ this.entityVersionCreatedFactory = entityVersionCreatedFactory;
++ this.entityDeletedFactory = entityDeletedFactory;
++ this.taskExecutor = taskExecutor;
++
+ this.collectionScope = collectionScope;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
}