You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/07/14 05:38:07 UTC
[01/50] usergrid git commit: Hook ReservationCache in via
ClusterClients
Repository: usergrid
Updated Branches:
refs/heads/master b0fbf14eb -> 849df6746
Hook ReservationCache in via ClusterClients
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2fb3ab32
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2fb3ab32
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2fb3ab32
Branch: refs/heads/master
Commit: 2fb3ab32bca20be4c9133e340cb97145ca452a3c
Parents: d12307b
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jun 24 11:34:53 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jun 24 11:34:53 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../actorsystem/ActorSystemManager.java | 36 ++++++++++++++-
.../actorsystem/ActorSystemManagerImpl.java | 24 +++++++++-
.../uniquevalues/ReservationCacheActor.java | 4 +-
.../uniquevalues/UniqueValueActor.java | 27 ++++++-----
.../uniquevalues/UniqueValuesServiceImpl.java | 47 ++++++++------------
.../collection/AbstractUniqueValueTest.java | 2 +-
7 files changed, 96 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2fb3ab32/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 060ec18..eca5927 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -155,7 +155,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
actorSystemManager.registerMessageType( UniqueValueActor.Cancellation.class, "/user/uvProxy" );
actorSystemManager.registerMessageType( UniqueValueActor.Confirmation.class, "/user/uvProxy" );
actorSystemManager.start();
- actorSystemManager.waitForClientActors();
+ actorSystemManager.waitForClientActor();
} catch (Throwable t) {
logger.error("Error starting Akka", t);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2fb3ab32/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
index c45ccac..cdb6caf 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
@@ -23,21 +23,55 @@ import akka.actor.ActorRef;
public interface ActorSystemManager {
+ /**
+ * Start create and start all Akka Actors, ClusterClients Routers and etc.
+ */
void start();
+ /**
+ * Start method used in JUnit tests.
+ */
void start(String hostname, Integer port, String currentRegion);
- void waitForClientActors();
+ /**
+ * Wait until ClientActor has seen some nodes and is ready for use.
+ */
+ void waitForClientActor();
+ /**
+ * True if ActorSystem and ClientActor are ready to be used.
+ */
boolean isReady();
+ /**
+ * MUST be called before start() to register any router producers to be configured.
+ */
void registerRouterProducer( RouterProducer routerProducer );
+ /**
+ * MUST be called before start() to register any messages to be sent.
+ * @param messageType Class of message.
+ * @param routerPath Router-path to which such messages are to be sent.
+ */
void registerMessageType( Class messageType, String routerPath );
+ /**
+ * Local client for ActorSystem, send all local messages here for routing.
+ */
ActorRef getClientActor();
+ /**
+ * Get ClientClient for specified remote region.
+ */
ActorRef getClusterClient(String region );
+ /**
+ * Get name of of this, the current region.
+ */
String getCurrentRegion();
+
+ /**
+ * Publish message to all topic subscribers in all regions.
+ */
+ void publishToAllRegions( String topic, Object message, ActorRef sender );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2fb3ab32/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index 5e23c20..89980bc 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -23,6 +23,8 @@ import akka.actor.*;
import akka.cluster.client.ClusterClient;
import akka.cluster.client.ClusterClientReceptionist;
import akka.cluster.client.ClusterClientSettings;
+import akka.cluster.pubsub.DistributedPubSub;
+import akka.cluster.pubsub.DistributedPubSubMediator;
import akka.pattern.Patterns;
import akka.util.Timeout;
import com.google.common.collect.ArrayListMultimap;
@@ -56,6 +58,8 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
private final Map<Class, String> routersByMessageType = new HashMap<>();
private final Map<String, ActorRef> clusterClientsByRegion = new HashMap<String, ActorRef>(20);
+ private ActorRef mediator;
+
private ActorRef clientActor;
private ListMultimap<String, String> seedsByRegion;
@@ -78,7 +82,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
this.port = null;
initAkka();
- waitForClientActors();
+ waitForClientActor();
}
@@ -132,6 +136,20 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
}
+ @Override
+ public void publishToAllRegions( String topic, Object message, ActorRef sender ) {
+
+ // send to local subscribers to topic
+ mediator.tell( new DistributedPubSubMediator.Publish( topic, message ), sender );
+
+ // send to each ClusterClient
+ for ( ActorRef clusterClient : clusterClientsByRegion.values() ) {
+ clusterClient.tell( new ClusterClient.Publish( topic, message ), sender );
+ }
+
+ }
+
+
private void initAkka() {
logger.info("Initializing Akka");
@@ -171,6 +189,8 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
for ( RouterProducer routerProducer : routerProducers ) {
routerProducer.createLocalSystemActors( localSystem );
}
+
+ mediator = DistributedPubSub.get( localSystem ).mediator();
}
@@ -359,7 +379,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
@Override
- public void waitForClientActors() {
+ public void waitForClientActor() {
waitForClientActor( clientActor );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2fb3ab32/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
index 2912c7d..51f5c8c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
@@ -49,7 +49,7 @@ public class ReservationCacheActor extends UntypedActor {
ReservationCache.getInstance().cacheReservation( res );
if ( ++reservationCount % 10 == 0 ) {
- logger.debug("Received {} reservations cache size {}",
+ logger.info("Received {} reservations cache size {}",
reservationCount, ReservationCache.getInstance().getSize());
}
@@ -58,7 +58,7 @@ public class ReservationCacheActor extends UntypedActor {
ReservationCache.getInstance().cancelReservation( can );
if ( ++cancellationCount % 10 == 0 ) {
- logger.debug("Received {} cancellations", cancellationCount);
+ logger.info("Received {} cancellations", cancellationCount);
}
} else if (msg instanceof DistributedPubSubMediator.SubscribeAck) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2fb3ab32/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
index e53710c..a14c63e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
@@ -21,6 +21,7 @@ import akka.actor.UntypedActor;
import akka.cluster.pubsub.DistributedPubSub;
import akka.cluster.pubsub.DistributedPubSubMediator;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
@@ -30,6 +31,7 @@ import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.util.UUID;
+
public class UniqueValueActor extends UntypedActor {
private static final Logger logger = LoggerFactory.getLogger( UniqueValueActor.class );
@@ -37,7 +39,7 @@ public class UniqueValueActor extends UntypedActor {
//private MetricsService metricsService;
- final private ActorRef mediator = DistributedPubSub.get(getContext().system()).mediator();
+ private final ActorSystemManager actorSystemManager;
private final UniqueValuesTable table;
@@ -48,6 +50,8 @@ public class UniqueValueActor extends UntypedActor {
// TODO: is there a way to avoid this ugly kludge? see also: ClusterSingletonRouter
this.table = UniqueValuesServiceImpl.injector.getInstance( UniqueValuesTable.class );
+ this.actorSystemManager = UniqueValuesServiceImpl.injector.getInstance( ActorSystemManager.class );
+
//logger.info("UniqueValueActor {} is live with table {}", name, table);
}
@@ -66,7 +70,7 @@ public class UniqueValueActor extends UntypedActor {
if ( message instanceof Reservation ) {
Reservation res = (Reservation) message;
-// final Timer.Context context = metricsService.getReservationTimer().time();
+ // final Timer.Context context = metricsService.getReservationTimer().time();
try {
Id owner = table.lookupOwner( res.getApplicationScope(), res.getOwner().getType(), res.getField() );
@@ -86,8 +90,7 @@ public class UniqueValueActor extends UntypedActor {
getSender().tell( new Response( Response.Status.IS_UNIQUE ), getSender() );
- mediator.tell( new DistributedPubSubMediator.Publish( "content",
- new Reservation( res ) ), getSelf() );
+ actorSystemManager.publishToAllRegions( "content", new Reservation( res ), getSelf() );
} catch (Throwable t) {
@@ -96,13 +99,13 @@ public class UniqueValueActor extends UntypedActor {
} finally {
-// context.stop();
+ // context.stop();
}
} else if ( message instanceof Confirmation) {
Confirmation con = (Confirmation) message;
-// final Timer.Context context = metricsService.getCommitmentTimer().time();
+ // final Timer.Context context = metricsService.getCommitmentTimer().time();
try {
Id owner = table.lookupOwner( con.getApplicationScope(), con.getOwner().getType(), con.getField() );
@@ -122,15 +125,14 @@ public class UniqueValueActor extends UntypedActor {
getSender().tell( new Response( Response.Status.IS_UNIQUE ), getSender() );
- mediator.tell( new DistributedPubSubMediator.Publish( "content",
- new Reservation( con ) ), getSelf() );
+ actorSystemManager.publishToAllRegions( "content", new Reservation( con ), getSelf() );
} catch (Throwable t) {
getSender().tell( new Response( Response.Status.ERROR ), getSender() );
logger.error( "Error processing request", t );
} finally {
-// context.stop();
+ // context.stop();
}
@@ -155,8 +157,7 @@ public class UniqueValueActor extends UntypedActor {
getSender().tell( new Response( Response.Status.SUCCESS ), getSender() );
- mediator.tell( new DistributedPubSubMediator.Publish( "content",
- new Reservation( can ) ), getSelf() );
+ actorSystemManager.publishToAllRegions( "content", new Reservation( can ), getSelf() );
} catch (Throwable t) {
getSender().tell( new Response( Response.Status.ERROR ), getSender() );
@@ -180,10 +181,12 @@ public class UniqueValueActor extends UntypedActor {
final String consistentHashKey;
public Request( ApplicationScope applicationScope, Id owner, UUID ownerVersion, Field field ) {
+
this.applicationScope = applicationScope;
this.owner = owner;
this.ownerVersion = ownerVersion;
this.field = field;
+
StringBuilder sb = new StringBuilder();
sb.append( applicationScope.getApplication() );
sb.append(":");
@@ -195,10 +198,12 @@ public class UniqueValueActor extends UntypedActor {
this.consistentHashKey = sb.toString();
}
public Request( Request req ) {
+
this.applicationScope = req.applicationScope;
this.owner = req.owner;
this.ownerVersion = req.ownerVersion;
this.field = req.field;
+
StringBuilder sb = new StringBuilder();
sb.append( req.applicationScope.getApplication() );
sb.append(":");
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2fb3ab32/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
index b888b1f..352c2e5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
@@ -82,17 +82,10 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
}
- // TODO: restore reservation cache
-// private void subscribeToReservations( ActorSystem localSystem, Map<String, ActorSystem> systemMap ) {
-//
-// for ( String region : systemMap.keySet() ) {
-// ActorSystem actorSystem = systemMap.get( region );
-// if ( !actorSystem.equals( localSystem ) ) {
-// logger.info("Starting ReservationCacheUpdater for {}", region );
-// actorSystem.actorOf( Props.create( ReservationCacheActor.class ), "subscriber");
-// }
-// }
-// }
+ private void subscribeToReservations( ActorSystem localSystem ) {
+ logger.info("Starting ReservationCacheUpdater");
+ localSystem.actorOf( Props.create( ReservationCacheActor.class ), "subscriber");
+ }
@Override
@@ -166,9 +159,9 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
scope, entity.getId(), version, field );
UniqueValueActor.Reservation res = reservationCache.get( request.getConsistentHashKey() );
-// if ( res != null ) {
-// getCacheCounter().inc();
-// }
+ // if ( res != null ) {
+ // getCacheCounter().inc();
+ // }
if ( res != null && !res.getOwner().equals( request.getOwner() )) {
throw new UniqueValueException( "Error property not unique (cache)", field);
}
@@ -194,10 +187,14 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
scope, entity.getId(), version, field );
if ( actorSystemManager.getCurrentRegion().equals( region ) ) {
+
+ // sending to current region, use local clientActor
ActorRef clientActor = actorSystemManager.getClientActor();
clientActor.tell( request, null );
} else {
+
+ // sending to remote region, send via cluster client for that region
ActorRef clusterClient = actorSystemManager.getClusterClient( region );
clusterClient.tell( new ClusterClient.Send("/user/clientActor", request), null );
}
@@ -205,16 +202,6 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
}
-// private ActorRef lookupRequestActorForType( String type ) {
-// final String region = getRegionsByType().get( type );
-// ActorRef requestActor = getRequestActorsByRegion().get( region == null ? currentRegion : region );
-// if ( requestActor == null ) {
-// throw new RuntimeException( "No request actor available for region: " + region );
-// }
-// return requestActor;
-// }
-
-
private void sendUniqueValueRequest(
Entity entity, String region, UniqueValueActor.Request request ) throws UniqueValueException {
@@ -226,19 +213,22 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
try {
Timeout t = new Timeout( 1, TimeUnit.SECONDS );
- // ask RequestActor and wait (up to timeout) for response
-
Future<Object> fut;
if ( actorSystemManager.getCurrentRegion().equals( region ) ) {
+
+ // sending to current region, use local clientActor
ActorRef clientActor = actorSystemManager.getClientActor();
fut = Patterns.ask( clientActor, request, t );
} else {
+
+ // sending to remote region, send via cluster client for that region
ActorRef clusterClient = actorSystemManager.getClusterClient( region );
fut = Patterns.ask( clusterClient, new ClusterClient.Send("/user/clientActor", request), t );
}
+ // wait (up to timeout) for response
response = (UniqueValueActor.Response) Await.result( fut, t.duration() );
if ( response != null && (
@@ -306,8 +296,7 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
@Override
public void createLocalSystemActors( ActorSystem localSystem ) {
- // TODO: restore reservation cache
- //subscribeToReservations( localSystem );
+ subscribeToReservations( localSystem );
}
@Override
@@ -317,6 +306,8 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
Map<String, Object> akka = (Map<String, Object>)configMap.get("akka");
+ // TODO: replace this configuration stuff with equivalent Java code in the above "create" methods
+
akka.put( "actor", new HashMap<String, Object>() {{
put( "deployment", new HashMap<String, Object>() {{
put( "/uvRouter/singleton/router", new HashMap<String, Object>() {{
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2fb3ab32/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
index 041835b..e7b4450 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
@@ -41,7 +41,7 @@ public class AbstractUniqueValueTest {
actorSystemManager.registerMessageType( UniqueValueActor.Cancellation.class, "/user/uvProxy" );
actorSystemManager.registerMessageType( UniqueValueActor.Confirmation.class, "/user/uvProxy" );
actorSystemManager.start( "127.0.0.1", port, "us-east" );
- actorSystemManager.waitForRequestActors();
+ actorSystemManager.waitForClientActor();
startedAkka.put( port, true );
}
[17/50] usergrid git commit: Merge branch 'release-2.1.1' into
usergrid-1268-akka-211
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' into usergrid-1268-akka-211
Conflicts:
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2d5ad05f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2d5ad05f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2d5ad05f
Branch: refs/heads/master
Commit: 2d5ad05f93a23cbe36046e1019d8b29ae06fe0e3
Parents: 639b421 8242878
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jun 30 15:41:48 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jun 30 15:41:48 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 15 ++-
.../corepersistence/CpRelationManager.java | 50 ++++++----
.../usergrid/persistence/EntityManager.java | 20 +++-
.../apache/usergrid/persistence/PathQuery.java | 3 +-
.../usergrid/persistence/RelationManager.java | 2 +
.../impl/EntityCollectionManagerImpl.java | 13 ++-
.../mvcc/stage/write/WriteUniqueVerify.java | 27 ++++--
.../UniqueValueSerializationStrategy.java | 25 ++++-
.../UniqueValueSerializationStrategyImpl.java | 97 +++++++++++---------
...iqueValueSerializationStrategyProxyImpl.java | 21 ++++-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 88 +++++++++++++++---
...niqueValueSerializationStrategyImplTest.java | 26 ++++--
.../collection/users/PermissionsResourceIT.java | 4 +-
.../services/AbstractCollectionService.java | 59 +++++++++---
.../services/AbstractConnectionsService.java | 5 +-
.../usergrid/services/ServiceInvocationIT.java | 28 +++++-
.../usergrid/tools/UniqueValueScanner.java | 69 +++++++-------
17 files changed, 392 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2d5ad05f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2d5ad05f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2d5ad05f/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2d5ad05f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2d5ad05f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 89431a7,0ebb6a1..f159096
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@@ -166,11 -121,12 +166,10 @@@ public class WriteUniqueVerify implemen
try {
-- // loading will retrieve the oldest unique value entry for the field
- UniqueValueSet set = uniqueValueStrat.load(
- scope, written.getEntityId().getType(), Collections.singletonList(written.getField()));
-
+ // don't use read repair on this pre-write check
+ UniqueValueSet set = uniqueValueStrat.load(scope, written.getEntityId().getType(),
+ Collections.singletonList(written.getField()), false);
-
set.forEach(uniqueValue -> {
if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2d5ad05f/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
index ccae9be,3d411a4..703d6c2
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
@@@ -18,21 -18,22 +18,17 @@@
package org.apache.usergrid.persistence.collection.mvcc.stage.write;
-import org.apache.usergrid.persistence.collection.FieldSet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
-import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
-import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
++import com.google.inject.Inject;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
- import org.apache.usergrid.persistence.collection.AbstractUniqueValueTest;
- import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueActor;
- import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
- import org.junit.Before;
- import org.junit.Rule;
- import org.junit.Test;
- import org.junit.runner.RunWith;
-
- import org.apache.usergrid.persistence.collection.EntityCollectionManager;
- import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
++import org.apache.usergrid.persistence.collection.*;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.mvcc.stage.TestEntityGenerator;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
++import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
++import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
++import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
++import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
@@@ -43,12 -44,11 +39,15 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.IntegerField;
import org.apache.usergrid.persistence.model.field.StringField;
-
-import com.google.inject.Inject;
++import org.apache.usergrid.persistence.model.util.UUIDGenerator;
++import org.junit.Before;
++import org.junit.Rule;
++import org.junit.Test;
++import org.junit.runner.RunWith;
- import com.google.inject.Inject;
-
- import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.Collections;
+import static junit.framework.TestCase.assertTrue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@@ -71,22 -68,11 +70,25 @@@ public class WriteUniqueVerifyIT extend
public MigrationManagerRule migrationManagerRule;
@Inject
+ public UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+
+ @Inject
public EntityCollectionManagerFactory cmf;
+ @Inject
+ ActorSystemManager actorSystemManager;
+
+ @Inject
+ UniqueValuesService uniqueValuesService;
+
+
+ @Before
+ public void initAkka() {
+ // each test class needs unique port number
+ initAkka( 2551, actorSystemManager, uniqueValuesService );
+ }
+
+
@Test
public void testConflict() {
@@@ -160,9 -148,69 +162,69 @@@
entity.setField(new StringField("name", "Alfa Romeo 8C Competizione", true));
entity.setField(new StringField("identifier", "ar8c", true));
entity.setField(new IntegerField("top_speed_mph", 182));
- entityManager.write( entity ).toBlocking().last();
+ entityManager.write( entity, null ).toBlocking().last();
entity.setField( new StringField("foo", "bar"));
- entityManager.write( entity ).toBlocking().last();
+ entityManager.write( entity, null ).toBlocking().last();
}
+
+ @Test
+ public void testConflictReadRepair() throws Exception {
+
+ final Id appId = new SimpleId("testNoConflict");
+
+
+
+ final ApplicationScope scope = new ApplicationScopeImpl( appId);
+
+ final EntityCollectionManager entityManager = cmf.createCollectionManager( scope );
+
+ final Entity entity = TestEntityGenerator.generateEntity();
+ entity.setField(new StringField("name", "Porsche 911 GT3", true));
+ entity.setField(new StringField("identifier", "911gt3", true));
+ entity.setField(new IntegerField("top_speed_mph", 194));
- entityManager.write( entity ).toBlocking().last();
++ entityManager.write( entity, null ).toBlocking().last();
+
+
+ FieldSet fieldSet =
+ entityManager.getEntitiesFromFields("test", Collections.singletonList(entity.getField("name")), true)
+ .toBlocking().last();
+
+ MvccEntity entityFetched = fieldSet.getEntity( entity.getField("name") );
+
+
+ final Entity entityDuplicate = TestEntityGenerator.generateEntity();
+ UniqueValue uniqueValue = new UniqueValueImpl(new StringField("name", "Porsche 911 GT3", true),
+ entityDuplicate.getId(), UUIDGenerator.newTimeUUID());
+
+ // manually insert a record to simulate a 'duplicate' trying to be inserted
+ uniqueValueSerializationStrategy.
+ write(scope, uniqueValue).execute();
+
+
+
+ FieldSet fieldSetAgain =
+ entityManager.getEntitiesFromFields("test", Collections.singletonList(entity.getField("name")), true)
+ .toBlocking().last();
+
+ MvccEntity entityFetchedAgain = fieldSetAgain.getEntity( entity.getField("name") );
+
+ assertEquals(entityFetched, entityFetchedAgain);
+
+
+ // now test writing the original entity again ( simulates a PUT )
+ // this should read repair and work
- entityManager.write( entity ).toBlocking().last();
++ entityManager.write( entity, null ).toBlocking().last();
+
+ FieldSet fieldSetAgainAgain =
+ entityManager.getEntitiesFromFields("test", Collections.singletonList(entity.getField("name")), true)
+ .toBlocking().last();
+
+ MvccEntity entityFetchedAgainAgain = fieldSetAgainAgain.getEntity( entity.getField("name") );
+
+ assertEquals(entityFetched, entityFetchedAgainAgain);
+
+
+
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2d5ad05f/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2d5ad05f/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
----------------------------------------------------------------------
[12/50] usergrid git commit: Clean up em.removeFromCollection and fix
UniqueValueScanner to just log duplicates if scanning all of unique values.
Posted by mr...@apache.org.
Clean up em.removeFromCollection and fix UniqueValueScanner to just log duplicates if scanning all of unique values.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/58039166
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/58039166
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/58039166
Branch: refs/heads/master
Commit: 58039166b9b119eb0766b473e43a8a732f3cbd00
Parents: 724968a
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jun 29 10:39:04 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jun 29 10:39:04 2016 -0700
----------------------------------------------------------------------
.../usergrid/tools/UniqueValueScanner.java | 69 +++++++++++---------
1 file changed, 38 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/58039166/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
index bdebd01..182e692 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
@@ -17,9 +17,7 @@
package org.apache.usergrid.tools;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.UUID;
+import java.util.*;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.Column;
@@ -208,9 +206,11 @@ public class UniqueValueScanner extends ToolBase {
UUID finalAppToFilter = appToFilter;
rows.forEachRemaining(row -> {
- String fieldName = row.getKey().getKey().getField().getName();
- String scopeType = row.getKey().getScope().getType();
- UUID scopeUUID = row.getKey().getScope().getUuid();
+ final String fieldName = row.getKey().getKey().getField().getName();
+ final String fieldValue = row.getKey().getKey().getField().getValue().toString();
+ final String scopeType = row.getKey().getScope().getType();
+ final UUID scopeUUID = row.getKey().getScope().getUuid();
+
if (!fieldName.equalsIgnoreCase(fieldType) ||
(finalAppToFilter != null && !finalAppToFilter.equals(scopeUUID))
@@ -223,16 +223,19 @@ public class UniqueValueScanner extends ToolBase {
em = emf.getEntityManager(scopeUUID);
}
- Iterator<Column<EntityVersion>> columns = row.getColumns().iterator();
- columns.forEachRemaining(column -> {
+ // if we have more than 1 column, let's check for a duplicate
+ if(row.getColumns().size() > 1) {
+
+ final List<EntityVersion> values = new ArrayList<>(row.getColumns().size());
+
+ Iterator<Column<EntityVersion>> columns = row.getColumns().iterator();
+ columns.forEachRemaining(column -> {
+
- EntityVersion entityVersion = column.getName();
- if (entityType != null &&
- entityVersion.getEntityId().getType().equalsIgnoreCase(entityType)
- ) {
+ final EntityVersion entityVersion = column.getName();
+
- String fieldValue = row.getKey().getKey().getField().getValue().toString();
logger.trace(
scopeType + ": " + scopeUUID + ", " +
@@ -242,30 +245,34 @@ public class UniqueValueScanner extends ToolBase {
);
- Entity entity = em.getUniqueEntityFromAlias(entityType, fieldValue, false);
+ if (entityType != null &&
+ entityVersion.getEntityId().getType().equalsIgnoreCase(entityType)
+ ) {
-// Optional<MvccEntity> entity = mvccEntitySerializationStrategy.
-// load(new ApplicationScopeImpl(new SimpleId(scopeUUID, scopeType)), entityVersion.getEntityId());
-//
-// if(!entity.isPresent()){
+ // add the first value into the list
+ if(values.size() == 0 ) {
- if (entity == null) {
+ values.add(entityVersion);
- logger.error("{}: {}. Entity with type=[{}], name=[{}], and uuid=[{}] has a unique value entry " +
- "but cannot be loaded from Mvcc entity serialization",
- scopeType,
- scopeUUID,
- entityVersion.getEntityId().getType(),
- fieldValue,
- entityVersion.getEntityId().getUuid());
- }
- } else {
- // do nothing
- }
+ }else{
+
+ if( !values.get(0).getEntityId().getUuid().equals(entityVersion.getEntityId().getUuid())){
+ values.add(entityVersion);
- });
+ logger.error("Duplicate found for field [{}={}]. Entry 1: [{}], Entry 2: [{}]",
+ fieldName, fieldValue, values.get(0).getEntityId(), entityVersion.getEntityId());
+
+ }
+
+ }
+
+
+ }
+
+ });
+ }
}
[33/50] usergrid git commit: Add support for later versions of tomcat.
Posted by mr...@apache.org.
Add support for later versions of tomcat.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4613e09e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4613e09e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4613e09e
Branch: refs/heads/master
Commit: 4613e09ee9d5b6df8fae3cc6419142d09a98c312
Parents: bd96b18
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jul 6 16:43:09 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jul 6 16:43:09 2016 -0700
----------------------------------------------------------------------
stack/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4613e09e/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index dd35891..1bf1f92 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -117,7 +117,7 @@
<shiro-version>1.2.4</shiro-version>
<slf4j-version>1.7.2</slf4j-version>
<snakeyaml-version>1.9</snakeyaml-version>
- <tomcat-version>7.0.64</tomcat-version>
+ <tomcat-version>8.5.3</tomcat-version>
<antlr.version>3.4</antlr.version>
<tika.version>1.4</tika.version>
<mockito.version>1.10.8</mockito.version>
[45/50] usergrid git commit: Fix issue where de-indexing assumed all
entities were time based UUIDs. They should be,
but there could be some legacy data. USERGRID-1305
Posted by mr...@apache.org.
Fix issue where de-indexing assumed all entities were time based UUIDs. They should be, but there could be some legacy data. USERGRID-1305
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f95bf96c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f95bf96c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f95bf96c
Branch: refs/heads/master
Commit: f95bf96cd30b41bc8c1edea73a827049acaee747
Parents: 86ae2cd
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jul 8 08:32:03 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jul 8 08:32:03 2016 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/index/IndexServiceImpl.java | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f95bf96c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 54b18bb..1d56928 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -278,10 +278,12 @@ public class IndexServiceImpl implements IndexService {
final EntityIndex ei = entityIndexFactory.
createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
-
+ // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code foe de-indexing
+ // previously .timstamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data)
final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId,
- entityId.getUuid().timestamp() ) );
+ Long.MAX_VALUE ) );
+
final EntityIndexBatch batch = ei.createBatch();
@@ -308,10 +310,11 @@ public class IndexServiceImpl implements IndexService {
final EntityIndex ei = entityIndexFactory.
createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
-
+ // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code foe de-indexing
+ // previously .timstamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data)
final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId,
- entityId.getUuid().timestamp() ) );
+ Long.MAX_VALUE ) );
final EntityIndexBatch batch = ei.createBatch();
[28/50] usergrid git commit: Set consistency level to ALL for now on
unique value serialization strategy as any cassandra inconsistencies will
break the usage of the unique value index.
Posted by mr...@apache.org.
Set consistency level to ALL for now on unique value serialization strategy as any cassandra inconsistencies will break the usage of the unique value index.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/02e1619c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/02e1619c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/02e1619c
Branch: refs/heads/master
Commit: 02e1619cbce1ada18d25048558ccdfc2a3341542
Parents: e0a5d98
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Jul 3 12:32:25 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Jul 3 12:32:25 2016 -0700
----------------------------------------------------------------------
.../collection/impl/EntityCollectionManagerImpl.java | 4 +++-
.../collection/mvcc/stage/write/WriteUniqueVerify.java | 5 +++--
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/02e1619c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 658c68c..5bdcf1d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -333,8 +333,10 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final UUID startTime = UUIDGenerator.newTimeUUID();
//Get back set of unique values that correspond to collection of fields
+ //Purposely use CL ALL as consistency is extremely important here, regardless of performance
UniqueValueSet set =
- uniqueValueSerializationStrategy.load( applicationScope, type, fields1 , uniqueIndexRepair);
+ uniqueValueSerializationStrategy
+ .load( applicationScope, ConsistencyLevel.CL_ALL, type, fields1 , uniqueIndexRepair);
//Short circuit if we don't have any uniqueValues from the given fields.
if ( !set.iterator().hasNext() ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/02e1619c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 53160dc..01e6621 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -123,8 +123,9 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
// loading will retrieve the oldest unique value entry for the field
// don't use read repair on this pre-write check
- UniqueValueSet set = uniqueValueStrat.load(scope, written.getEntityId().getType(),
- Collections.singletonList(written.getField()), false);
+ // use CL ALL as consistency is extremely important here, more so than performance
+ UniqueValueSet set = uniqueValueStrat.load(scope, ConsistencyLevel.CL_ALL,
+ written.getEntityId().getType(), Collections.singletonList(written.getField()), false);
set.forEach(uniqueValue -> {
[09/50] usergrid git commit: Add missing property
Posted by mr...@apache.org.
Add missing property
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ca9383b2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ca9383b2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ca9383b2
Branch: refs/heads/master
Commit: ca9383b21efd17cc8c20314af12046e036e28be7
Parents: 14f5f7a
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jun 29 08:44:32 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jun 29 08:44:32 2016 -0400
----------------------------------------------------------------------
stack/config/src/main/resources/usergrid-default.properties | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ca9383b2/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 40557ba..29b8d36 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -439,6 +439,9 @@ collection.akka.region.seeds=
# Region MUST be in the region list specified in the 'usergrid.queue.regionList' property
collection.akka.authoritative.region=
+# Default number of Akka actors to start per instance / router producer
+collection.akka.instances-per-node=300
+
######################### Usergrid Unique Values Validation ##################
#
[29/50] usergrid git commit: Improve consistency when returning
entities/ids loaded from unique valuie index when entity repair ( removal of
stale entities) has been executed.
Posted by mr...@apache.org.
Improve consistency when returning entities/ids loaded from unique valuie index when entity repair ( removal of stale entities) has been executed.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8982ddd3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8982ddd3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8982ddd3
Branch: refs/heads/master
Commit: 8982ddd3a12541e988e802c3fc1dd5feca44ad6d
Parents: 02e1619
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Jul 3 15:25:22 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Jul 3 15:25:22 2016 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 44 ++++++++++++++------
.../persistence/collection/FieldSet.java | 12 ++++++
.../impl/EntityCollectionManagerImpl.java | 23 +---------
.../serialization/SerializationFig.java | 4 --
.../serialization/impl/MutableFieldSet.java | 12 ++++++
5 files changed, 57 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 1ed05bd..318bba2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -860,8 +860,6 @@ public class CpEntityManager implements EntityManager {
Timer.Context repairedEntityGet = entGetRepairedEntityTimer.time();
- // TODO: can't we just sub in the getEntityRepair method here
- // so for every read of a uniqueEntityField we can verify it is correct?
StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString());
@@ -877,11 +875,22 @@ public class CpEntityManager implements EntityManager {
return null;
}
- FieldSet fieldSet = fieldSetObservable
- .doOnError( t ->
- logger.error("Unable to retrieve unique values due to: {}", t.getMessage())
- )
- .toBlocking().last();
+ FieldSet fieldSet = fieldSetObservable.toBlocking().last();
+
+ // do a re-load if we know an entity repair was executed
+ // a hit to performance, but we need to return consistent success respones if we're repairing data
+ if(fieldSet.getEntityRepairExecuted()){
+
+ if(logger.isTraceEnabled()){
+ logger.trace("One or more entities were repaired ( removed ) during loading of unique field [{}={}], " +
+ "executing the unique value lookup once more for consistency", uniqueLookupRepairField.getName(),
+ uniqueLookupRepairField.getValue());
+ }
+
+ fieldSet = ecm.getEntitiesFromFields(
+ Inflector.getInstance().singularize( collectionType ),
+ Collections.singletonList(uniqueLookupRepairField), uniqueIndexRepair).toBlocking().last();
+ }
repairedEntityGet.stop();
@@ -912,11 +921,22 @@ public class CpEntityManager implements EntityManager {
return null;
}
- FieldSet fieldSet = fieldSetObservable
- .doOnError( t ->
- logger.error("Unable to retrieve unique values due to: {}", t.getMessage())
- )
- .toBlocking().last();
+ FieldSet fieldSet = fieldSetObservable.toBlocking().last();
+
+ // do a re-load if we know an entity repair was executed
+ // a hit to performance, but we need to return consistent success respones if we're repairing data
+ if(fieldSet.getEntityRepairExecuted()){
+
+ if(logger.isTraceEnabled()){
+ logger.trace("One or more entities were repaired ( removed ) during loading of unique field [{}={}], " +
+ "executing the unique value lookup once more for consistency", uniqueLookupRepairField.getName(),
+ uniqueLookupRepairField.getValue());
+ }
+
+ fieldSet = ecm.getEntitiesFromFields(
+ Inflector.getInstance().singularize( collectionType ),
+ Collections.singletonList(uniqueLookupRepairField), uniqueIndexRepair).toBlocking().last();
+ }
if(fieldSet == null || fieldSet.isEmpty()) {
return null;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
index 43f470d..1fabd5a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
@@ -45,4 +45,16 @@ public interface FieldSet {
* @return
*/
public boolean isEmpty();
+
+ /**
+ * Return true if there was at least entity repaired ( removed ) during the building of this set.
+ * @return
+ */
+ public boolean getEntityRepairExecuted();
+
+ /**
+ * Return true if there was at least entity repaired ( removed ) during the building of this set.
+ * @return
+ */
+ public void setEntityRepairExecuted(final boolean entityRepairExecuted);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 5bdcf1d..9261b56 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -413,32 +413,11 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
if ( deleteBatch.getRowCount() > 0 ) {
+ response.setEntityRepairExecuted(true);
deleteBatch.execute();
-
- // optionally sleep after read repair as some tasks immediately try to write after the delete
- if ( serializationFig.getReadRepairDelay() > 0 ){
-
- try {
-
- if(logger.isTraceEnabled()) {
- logger.trace("Sleeping {}ms after unique value read repair execution",
- serializationFig.getReadRepairDelay());
- }
-
- Thread.sleep(Math.min(serializationFig.getReadRepairDelay(), 200L));
-
- } catch (InterruptedException e) {
-
- // do nothing if sleep fails; log and continue on
- logger.warn("Sleep during unique value read repair failed.");
- }
-
- }
-
}
-
return response;
}
catch ( ConnectionException e ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 12033fe..bdd71c7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -58,8 +58,4 @@ public interface SerializationFig extends GuicyFig {
@Default( "150" )
int getUniqueVerifyPoolSize();
-
- @Key ( "collection.readrepair.delay" )
- @Default( "0" ) // in milliseconds
- int getReadRepairDelay();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
index df2b850..da874c8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
@@ -31,6 +31,8 @@ public class MutableFieldSet implements FieldSet {
private final Map<Field<?>, MvccEntity> entities;
+ private boolean readRepairExecuted;
+
public MutableFieldSet( final int expectedSize ) {
this.entities = new HashMap<>( expectedSize );
@@ -60,4 +62,14 @@ public class MutableFieldSet implements FieldSet {
public boolean isEmpty() {
return entities.size() == 0;
}
+
+ @Override
+ public void setEntityRepairExecuted( final boolean readRepairExecuted ){
+ this.readRepairExecuted = readRepairExecuted;
+ }
+
+ @Override
+ public boolean getEntityRepairExecuted(){
+ return readRepairExecuted;
+ }
}
[19/50] usergrid git commit: Add additional logging during write
unique verification.
Posted by mr...@apache.org.
Add additional logging during write unique verification.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/57231f0d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/57231f0d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/57231f0d
Branch: refs/heads/master
Commit: 57231f0d6594916faa459164cabbce9bfb9ffa4d
Parents: b0c3059
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Jun 30 21:05:35 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Jun 30 21:05:35 2016 -0700
----------------------------------------------------------------------
.../collection/mvcc/stage/write/WriteUniqueVerify.java | 12 ++++++++++++
.../impl/UniqueValueSerializationStrategyImpl.java | 3 ++-
2 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/57231f0d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 0ebb6a1..7f32888 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -131,6 +131,18 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
+ if(logger.isTraceEnabled()){
+ logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and" +
+ "entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " +
+ "entity version [{}]",
+ written.getField().getName(),
+ written.getField().getValue().toString(),
+ written.getEntityId().getUuid(),
+ written.getEntityVersion(),
+ uniqueValue.getEntityId().getUuid(),
+ uniqueValue.getEntityVersion());
+ }
+
preWriteUniquenessViolations.put(field.getName(), field);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/57231f0d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index ce1f20e..f37e219 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -358,7 +358,8 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
delete(appScope, candidate).execute();
} catch (ConnectionException e) {
- // do nothing for now
+ logger.error( "Unable to connect to cassandra during duplicate repair of [{}={}]",
+ field.getName(), field.getValue().toString() );
}
});
[42/50] usergrid git commit: Should be ignored
Posted by mr...@apache.org.
Should be ignored
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2fb55f05
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2fb55f05
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2fb55f05
Branch: refs/heads/master
Commit: 2fb55f05da923b9812f18a30952ab338d1822b70
Parents: d5d89a8
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 15:09:57 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 15:09:57 2016 -0400
----------------------------------------------------------------------
.../test/java/org/apache/usergrid/rest/UniqueCatsIT.java | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2fb55f05/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
index c0209af..6561389 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
@@ -63,7 +63,7 @@ public class UniqueCatsIT {
private static final AtomicInteger dupCounter = new AtomicInteger( 0 );
@Test
- //@Ignore("Intended for use against prod-like cluster")
+ @Ignore("Intended for use against prod-like cluster")
public void testDuplicatePrevention() throws Exception {
int numThreads = 20;
@@ -83,12 +83,7 @@ public class UniqueCatsIT {
final AtomicBoolean failed = new AtomicBoolean(false);
- //String[] targetHosts = {"http://localhost:8080"};
-
- String[] targetHosts = {
- "https://ug21-west.e2e.apigee.net",
- "https://ug21-east.e2e.apigee.net"
- };
+ String[] targetHosts = {"http://localhost:8080"};
for (int i = 0; i < numCats; i++) {
[03/50] usergrid git commit: Merge branch 'release-2.1.1' into
usergrid-1268-akka-211
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' into usergrid-1268-akka-211
Conflicts:
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/25303e6d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/25303e6d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/25303e6d
Branch: refs/heads/master
Commit: 25303e6d04faf47162c8ac94466deec178b45326
Parents: 2fb3ab3 c4f72ef
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jun 24 15:04:22 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jun 24 15:04:22 2016 -0400
----------------------------------------------------------------------
deployment/pcf/LICENSE | 201 ++++++++
deployment/pcf/README.md | 44 ++
deployment/pcf/addBlobs.sh | 25 +
deployment/pcf/apache-usergrid-tile-1.6.yml | 510 +++++++++++++++++++
deployment/pcf/config/blobs.yml | 19 +
deployment/pcf/config/final.yml | 24 +
deployment/pcf/content_migrations.yml | 53 ++
deployment/pcf/createRelease.sh | 24 +
deployment/pcf/createTile.sh | 32 ++
deployment/pcf/jobs/delete-all/monit | 17 +
deployment/pcf/jobs/delete-all/spec | 44 ++
.../jobs/delete-all/templates/delete-all.sh.erb | 131 +++++
deployment/pcf/jobs/deploy-all/monit | 18 +
deployment/pcf/jobs/deploy-all/spec | 145 ++++++
.../jobs/deploy-all/templates/deploy-all.sh.erb | 384 ++++++++++++++
.../pcf/jobs/docker-bosh-cassandra_docker/monit | 23 +
.../pcf/jobs/docker-bosh-cassandra_docker/spec | 26 +
.../docker-bosh-cassandra_docker.sh.erb | 69 +++
.../jobs/docker-bosh-elasticsearch_docker/monit | 23 +
.../jobs/docker-bosh-elasticsearch_docker/spec | 26 +
.../docker-bosh-elasticsearch_docker.sh.erb | 69 +++
.../pcf/packages/cassandra_docker/packaging | 26 +
deployment/pcf/packages/cassandra_docker/spec | 26 +
deployment/pcf/packages/cf_cli/packaging | 26 +
deployment/pcf/packages/cf_cli/spec | 25 +
deployment/pcf/packages/common/packaging | 23 +
deployment/pcf/packages/common/spec | 25 +
.../pcf/packages/elasticsearch_docker/packaging | 26 +
.../pcf/packages/elasticsearch_docker/spec | 26 +
deployment/pcf/packages/usergrid_app/packaging | 27 +
deployment/pcf/packages/usergrid_app/spec | 27 +
deployment/pcf/run.sh | 25 +
deployment/pcf/src/common/utils.sh | 107 ++++
deployment/pcf/src/templates/all_open.json | 6 +
deployment/pcf/src/usergrid_app/manifest.yml | 22 +
docs/orgs-and-apps/application.md | 104 ++++
.../main/resources/usergrid-default.properties | 4 +
.../corepersistence/CpEntityManager.java | 36 +-
.../corepersistence/CpRelationManager.java | 32 +-
.../asyncevents/AsyncEventServiceImpl.java | 34 +-
.../usergrid/persistence/EntityManager.java | 2 +-
.../collection/EntityCollectionManager.java | 3 +-
.../impl/EntityCollectionManagerImpl.java | 34 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 48 +-
.../UniqueValueSerializationStrategy.java | 3 +-
.../UniqueValueSerializationStrategyImpl.java | 139 ++++-
.../collection/EntityCollectionManagerIT.java | 6 +-
...niqueValueSerializationStrategyImplTest.java | 218 +++++++-
.../graph/impl/GraphManagerImpl.java | 4 +
stack/pom.xml | 2 -
.../security/SecuredResourceFilterFactory.java | 72 ++-
.../collection/devices/DevicesResourceIT.java | 3 +-
.../usergrid/rest/management/AdminUsersIT.java | 80 +++
.../cassandra/ManagementServiceImpl.java | 33 +-
.../services/AbstractCollectionService.java | 49 +-
.../services/AbstractConnectionsService.java | 6 +-
.../services/devices/DevicesService.java | 2 +
stack/tools/pom.xml | 1 +
.../org/apache/usergrid/tools/ToolBase.java | 28 +-
.../usergrid/tools/UniqueValueScanner.java | 276 ++++++++++
.../main/resources/toolsApplicationContext.xml | 7 +
61 files changed, 3446 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 848ed2e,7b76dc8..92ea6aa
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@@ -47,15 -39,20 +47,12 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.util.EntityUtils;
-
-import com.google.common.base.Preconditions;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import com.netflix.astyanax.model.ConsistencyLevel;
-import com.netflix.hystrix.HystrixCommand;
-import com.netflix.hystrix.HystrixCommandGroupKey;
-import com.netflix.hystrix.HystrixThreadPoolProperties;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import rx.functions.Action1;
- import java.util.ArrayList;
- import java.util.HashMap;
- import java.util.List;
- import java.util.Map;
++import java.util.*;
+
/**
* This phase execute all unique value verification on the MvccEntity.
@@@ -163,6 -119,28 +161,29 @@@ public class WriteUniqueVerify implemen
// use write-first then read strategy
final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
+ try {
+
+ // loading will retrieve the oldest unique value entry for the field
- UniqueValueSet set = uniqueValueStrat.load(scope, written.getEntityId().getType(), Collections.singletonList(written.getField()));
++ UniqueValueSet set = uniqueValueStrat.load(
++ scope, written.getEntityId().getType(), Collections.singletonList(written.getField()));
+
+
+ set.forEach(uniqueValue -> {
+
+ if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
+
+ preWriteUniquenessViolations.put(field.getName(), field);
+
+ }
+
+ });
+
+
+ } catch (ConnectionException e) {
+
+ throw new RuntimeException("Error connecting to cassandra", e);
+ }
+
// use TTL in case something goes wrong before entity is finally committed
final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
@@@ -226,7 -207,7 +252,8 @@@
@Override
protected Map<String, Field> getFallback() {
-- // fallback with same CL as there are many reasons the 1st execution failed, not just due to consistency problems
++ // fallback with same CL as there are many reasons the 1st execution failed,
++ // not just due to consistency problems
return executeStrategy(fig.getReadCL());
}
@@@ -257,6 -239,16 +285,16 @@@
final Id returnedEntityId = uniqueValue.getEntityId();
if ( !entity.getId().equals(returnedEntityId) ) {
+
+ if(logger.isTraceEnabled()) {
- logger.trace("Violation occurred when verifying unique value [{}={}]. Returned entity id [{}] " +
- "does not match expected entity id [{}]",
++ logger.trace("Violation occurred when verifying unique value [{}={}]. " +
++ "Returned entity id [{}] does not match expected entity id [{}]",
+ field.getName(), field.getValue().toString(),
+ returnedEntityId,
+ entity.getId()
+ );
+ }
+
uniquenessViolations.put( field.getName(), field );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index bb2377a,0f3daa4..f8322c6
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@@ -29,8 -34,6 +29,7 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
- import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueActor;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
import org.apache.usergrid.persistence.collection.util.EntityHelper;
import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.guicyfig.SetConfigTestBypass;
@@@ -46,19 -49,20 +45,18 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.field.IntegerField;
import org.apache.usergrid.persistence.model.field.StringField;
-
-import com.fasterxml.uuid.UUIDComparator;
-import com.google.inject.Inject;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
import rx.Observable;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
- import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.Assert.*;
/** @author tnine */
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25303e6d/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
----------------------------------------------------------------------
[18/50] usergrid git commit: Add additional logging during unique
value read repair.
Posted by mr...@apache.org.
Add additional logging during unique value read repair.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b0c30593
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b0c30593
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b0c30593
Branch: refs/heads/master
Commit: b0c30593a24d23fedbc16e033b91fb8cb1d999ea
Parents: 8242878
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Jun 30 20:36:01 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Jun 30 20:36:01 2016 -0700
----------------------------------------------------------------------
.../impl/UniqueValueSerializationStrategyImpl.java | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b0c30593/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index 4456123..ce1f20e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -335,7 +335,9 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
// do nothing, only versions can be newer and we're not worried about newer versions of same entity
if (logger.isTraceEnabled()) {
- logger.trace("Candidate unique value is equal to the current unique value");
+ logger.trace("Current unique value [{}={}] entry has UUID [{}] equal to candidate UUID [{}]",
+ field.getName(), field.getValue().toString(), uniqueValue.getEntityId().getUuid(),
+ candidates.get(candidates.size() -1));
}
// update candidate w/ latest version
@@ -365,7 +367,8 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
candidates.clear();
if (logger.isTraceEnabled()) {
- logger.trace("Updating candidate to entity id [{}] and entity version [{}]",
+ logger.trace("Updating candidate unique value [{}={}] to entity id [{}] and " +
+ "entity version [{}]", field.getName(), field.getValue().toString(),
uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion());
}
@@ -390,6 +393,10 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
}
// take the last candidate ( should be the latest version) and add to the result set
+ if(logger.isTraceEnabled()){
+ logger.trace("Adding unique value [{}] to response set.",
+ candidates.get(candidates.size() -1));
+ }
uniqueValueSet.addValue(candidates.get(candidates.size() -1));
}
[27/50] usergrid git commit: Add more logging to unique value
serialization.
Posted by mr...@apache.org.
Add more logging to unique value serialization.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e0a5d981
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e0a5d981
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e0a5d981
Branch: refs/heads/master
Commit: e0a5d9811f7851263336c04302b0208c64573a73
Parents: ada914a
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Jul 3 10:52:31 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Jul 3 10:52:31 2016 -0700
----------------------------------------------------------------------
.../mvcc/stage/write/WriteUniqueVerify.java | 2 +-
.../UniqueValueSerializationStrategyImpl.java | 22 ++++++++++++++++++++
2 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0a5d981/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 538160a..53160dc 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -132,7 +132,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
if(logger.isTraceEnabled()){
- logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and" +
+ logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and " +
"entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " +
"entity version [{}]",
written.getField().getName(),
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0a5d981/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index 8f304f3..8c1f2d2 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -278,6 +278,12 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
.withColumnRange(new RangeBuilder().setLimit(serializationFig.getMaxLoadSize()).build())
.execute().getResult().iterator();
+ if( !results.hasNext()){
+ if(logger.isTraceEnabled()){
+ logger.trace("No partitions returned for unique value lookup");
+ }
+ }
+
while ( results.hasNext() )
@@ -291,6 +297,10 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
//sanity check, nothing to do, skip it
if ( !columnList.hasNext() ) {
+ if(logger.isTraceEnabled()){
+ logger.trace("No cells exist in partition for unique value [{}={}]",
+ field.getName(), field.getValue().toString());
+ }
continue;
}
@@ -317,12 +327,24 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
// set the initial candidate and move on
if (candidates.size() == 0) {
candidates.add(uniqueValue);
+
+ if (logger.isTraceEnabled()) {
+ logger.trace("First entry for unique value [{}={}] found for application [{}], adding " +
+ "entry with entity id [{}] and entity version [{}] to the candidate list and continuing",
+ field.getName(), field.getValue().toString(), applicationId.getType(),
+ uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion());
+ }
+
continue;
}
if(!useReadRepair){
// take only the first
+ if (logger.isTraceEnabled()) {
+ logger.trace("Read repair not enabled for this request of unique value [{}={}], breaking out" +
+ " of cell loop", field.getName(), field.getValue().toString());
+ }
break;
} else {
[11/50] usergrid git commit: Merge branch 'release-2.1.1' into
usergrid-1268-akka-211
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' into usergrid-1268-akka-211
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/639b421c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/639b421c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/639b421c
Branch: refs/heads/master
Commit: 639b421cdaa585f877ad1c50e62fa36c52384762
Parents: ca9383b 37c482c
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jun 29 08:45:45 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jun 29 08:45:45 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 40 ++++++++---------
.../asyncevents/EventBuilderImpl.java | 7 ++-
.../UniqueValueSerializationStrategyImpl.java | 11 ++---
.../index/impl/EsEntityIndexImpl.java | 47 +++++++-------------
4 files changed, 43 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/639b421c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
[02/50] usergrid git commit: Merge branch 'release-2.1.1' of
https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c4f72ef1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c4f72ef1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c4f72ef1
Branch: refs/heads/master
Commit: c4f72ef14b4005f7ac7316e43e84ae048054203a
Parents: cb90c0b e982ba4
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jun 24 14:31:29 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jun 24 14:31:29 2016 -0400
----------------------------------------------------------------------
deployment/pcf/LICENSE | 201 ++++++++
deployment/pcf/README.md | 44 ++
deployment/pcf/addBlobs.sh | 25 +
deployment/pcf/apache-usergrid-tile-1.6.yml | 510 +++++++++++++++++++
deployment/pcf/config/blobs.yml | 19 +
deployment/pcf/config/final.yml | 24 +
deployment/pcf/content_migrations.yml | 53 ++
deployment/pcf/createRelease.sh | 24 +
deployment/pcf/createTile.sh | 32 ++
deployment/pcf/jobs/delete-all/monit | 17 +
deployment/pcf/jobs/delete-all/spec | 44 ++
.../jobs/delete-all/templates/delete-all.sh.erb | 131 +++++
deployment/pcf/jobs/deploy-all/monit | 18 +
deployment/pcf/jobs/deploy-all/spec | 145 ++++++
.../jobs/deploy-all/templates/deploy-all.sh.erb | 384 ++++++++++++++
.../pcf/jobs/docker-bosh-cassandra_docker/monit | 23 +
.../pcf/jobs/docker-bosh-cassandra_docker/spec | 26 +
.../docker-bosh-cassandra_docker.sh.erb | 69 +++
.../jobs/docker-bosh-elasticsearch_docker/monit | 23 +
.../jobs/docker-bosh-elasticsearch_docker/spec | 26 +
.../docker-bosh-elasticsearch_docker.sh.erb | 69 +++
.../pcf/packages/cassandra_docker/packaging | 26 +
deployment/pcf/packages/cassandra_docker/spec | 26 +
deployment/pcf/packages/cf_cli/packaging | 26 +
deployment/pcf/packages/cf_cli/spec | 25 +
deployment/pcf/packages/common/packaging | 23 +
deployment/pcf/packages/common/spec | 25 +
.../pcf/packages/elasticsearch_docker/packaging | 26 +
.../pcf/packages/elasticsearch_docker/spec | 26 +
deployment/pcf/packages/usergrid_app/packaging | 27 +
deployment/pcf/packages/usergrid_app/spec | 27 +
deployment/pcf/run.sh | 25 +
deployment/pcf/src/common/utils.sh | 107 ++++
deployment/pcf/src/templates/all_open.json | 6 +
deployment/pcf/src/usergrid_app/manifest.yml | 22 +
.../main/resources/usergrid-default.properties | 4 +
.../corepersistence/CpEntityManager.java | 38 +-
.../corepersistence/CpRelationManager.java | 32 +-
.../corepersistence/EntityManagerFig.java | 2 +-
.../asyncevents/AsyncEventService.java | 6 +-
.../asyncevents/AsyncEventServiceImpl.java | 45 +-
.../asyncevents/EventBuilder.java | 6 +-
.../asyncevents/EventBuilderImpl.java | 46 +-
.../model/DeIndexOldVersionsEvent.java | 12 +-
.../corepersistence/index/IndexService.java | 23 +-
.../corepersistence/index/IndexServiceImpl.java | 86 ++--
.../pipeline/cursor/CursorSerializerUtil.java | 9 -
.../pipeline/cursor/RequestCursor.java | 29 +-
.../usergrid/persistence/EntityManager.java | 2 +-
.../persistence/entities/Notification.java | 38 +-
.../collection/EntityCollectionManager.java | 13 +-
.../impl/EntityCollectionManagerImpl.java | 72 ++-
.../mvcc/stage/write/WriteUniqueVerify.java | 44 +-
.../serialization/SerializationFig.java | 5 +
.../UniqueValueSerializationStrategy.java | 3 +-
.../serialization/impl/LogEntryIterator.java | 128 +++++
.../UniqueValueSerializationStrategyImpl.java | 139 ++++-
.../collection/EntityCollectionManagerIT.java | 5 +-
...niqueValueSerializationStrategyImplTest.java | 218 +++++++-
.../graph/impl/GraphManagerImpl.java | 4 +
.../usergrid/persistence/index/EntityIndex.java | 5 +-
.../usergrid/persistence/index/IndexFig.java | 2 +-
.../index/impl/EsEntityIndexImpl.java | 64 +--
.../persistence/index/impl/EntityIndexTest.java | 41 --
stack/pom.xml | 2 -
.../security/SecuredResourceFilterFactory.java | 72 ++-
.../collection/CollectionsResourceIT.java | 87 +++-
.../collection/devices/DevicesResourceIT.java | 3 +-
.../usergrid/rest/management/AdminUsersIT.java | 80 +++
.../resource/endpoints/CollectionEndpoint.java | 24 +-
.../cassandra/ManagementServiceImpl.java | 33 +-
.../services/AbstractCollectionService.java | 49 +-
.../services/AbstractConnectionsService.java | 6 +-
.../services/devices/DevicesService.java | 2 +
.../impl/ApplicationQueueManagerImpl.java | 52 +-
.../AbstractServiceNotificationIT.java | 26 +-
stack/tools/pom.xml | 1 +
.../org/apache/usergrid/tools/ToolBase.java | 28 +-
.../usergrid/tools/UniqueValueScanner.java | 276 ++++++++++
.../main/resources/toolsApplicationContext.xml | 7 +
80 files changed, 3835 insertions(+), 358 deletions(-)
----------------------------------------------------------------------
[05/50] usergrid git commit: Allow Akka authoritative region to be
null and if null, uses current region as authoritative region.
Posted by mr...@apache.org.
Allow Akka authoritative region to be null and if null, uses current region as authoritative region.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/14f5f7ab
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/14f5f7ab
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/14f5f7ab
Branch: refs/heads/master
Commit: 14f5f7abd4e9275ce1aba3bfb1b92c97a780bf96
Parents: 25303e6
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Jun 27 12:53:39 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Jun 27 12:53:39 2016 -0400
----------------------------------------------------------------------
.../usergrid/persistence/actorsystem/ActorSystemManagerImpl.java | 4 +++-
.../persistence/collection/mvcc/stage/write/WriteCommit.java | 3 +++
.../collection/mvcc/stage/write/WriteUniqueVerify.java | 3 +++
3 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/14f5f7ab/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index 89980bc..05f837d 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -172,7 +172,9 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
}
if ( StringUtils.isEmpty( actorSystemFig.getAkkaAuthoritativeRegion() )) {
- throw new RuntimeException( "No value specified for " + ActorSystemFig.AKKA_AUTHORITATIVE_REGION);
+ logger.warn("No value for {} specified, will use current region as authoriative region",
+ ActorSystemFig.AKKA_AUTHORITATIVE_REGION);
+ //throw new RuntimeException( "No value specified for " + ActorSystemFig.AKKA_AUTHORITATIVE_REGION);
}
List regionList = Arrays.asList( actorSystemFig.getRegionList().toLowerCase().split(",") );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/14f5f7ab/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
index 267e793..65d1734 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
@@ -135,6 +135,9 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Collect
if ( region == null ) {
region = actorSystemFig.getAkkaAuthoritativeRegion();
}
+ if ( region == null ) {
+ region = actorSystemFig.getRegion();
+ }
confirmUniqueFieldsAkka( mvccEntity, version, applicationScope, region );
} else {
confirmUniqueFields( mvccEntity, version, applicationScope, logMutation );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/14f5f7ab/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 92ea6aa..89431a7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -123,6 +123,9 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
if ( region == null ) {
region = actorSystemFig.getAkkaAuthoritativeRegion();
}
+ if ( region == null ) {
+ region = actorSystemFig.getRegion();
+ }
try {
akkaUvService.reserveUniqueValues( applicationScope, entity, mvccEntity.getVersion(), region );
[07/50] usergrid git commit: Enhance some logic around deleting of
edges.
Posted by mr...@apache.org.
Enhance some logic around deleting of edges.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9c4b5242
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9c4b5242
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9c4b5242
Branch: refs/heads/master
Commit: 9c4b5242a169cb4e90922c3fc7755be3bdac8be8
Parents: 09280b1
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Jun 27 21:13:45 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Jun 27 21:13:45 2016 -0700
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 40 ++++++++---------
.../asyncevents/EventBuilderImpl.java | 7 ++-
.../index/impl/EsEntityIndexImpl.java | 47 +++++++-------------
3 files changed, 37 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9c4b5242/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 39b3161..6e1bade 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -524,44 +524,42 @@ public class CpRelationManager implements RelationManager {
}
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
- org.apache.usergrid.persistence.model.entity.Entity memberEntity = ( ( CpEntityManager ) em ).load( entityId );
-
// remove edge from collection to item
GraphManager gm = managerCache.getGraphManager( applicationScope );
- List<Edge> removedEdges = new ArrayList<>();
- //run our delete
- gm.loadEdgeVersions(
- CpNamingUtils.createEdgeFromCollectionName( cpHeadEntity.getId(), collectionName, memberEntity.getId() ) )
- .flatMap(edge -> gm.markEdge(edge)).flatMap(edge -> gm.deleteEdge(edge))
- .doOnNext(edge -> removedEdges.add(edge)).toBlocking()
- .lastOrDefault(null);
+ // mark the edge versions and take the first for later delete edge queue event ( load is descending )
+ final Edge markedSourceEdge = gm.loadEdgeVersions(
+ CpNamingUtils.createEdgeFromCollectionName( cpHeadEntity.getId(), collectionName, entityId ) )
+ .flatMap(edge -> gm.markEdge(edge)).toBlocking().firstOrDefault(null);
+
+
+ Edge markedReversedEdge = null;
CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName );
if (collection != null && collection.getLinkedCollection() != null) {
// delete reverse edges
final String pluralType = InflectionUtils.pluralize( cpHeadEntity.getId().getType() );
- gm.loadEdgeVersions(
- CpNamingUtils.createEdgeFromCollectionName( memberEntity.getId(), pluralType, cpHeadEntity.getId() ) )
- .flatMap(reverseEdge -> gm.markEdge(reverseEdge))
- .flatMap(reverseEdge -> gm.deleteEdge(reverseEdge))
- .doOnNext(reverseEdge -> removedEdges.add(reverseEdge))
- .toBlocking().lastOrDefault(null);
+ markedReversedEdge = gm.loadEdgeVersions(
+ CpNamingUtils.createEdgeFromCollectionName( entityId, pluralType, cpHeadEntity.getId() ) )
+ .flatMap(reverseEdge -> gm.markEdge(reverseEdge)).toBlocking().firstOrDefault(null);
}
/**
- * Remove from the index
+ * Remove from the index. This will call gm.deleteEdge which also deletes the reverse edge(s) and de-indexes
+ * older versions of the edge(s).
*
*/
+ if( markedSourceEdge != null ) {
+ indexService.queueDeleteEdge(applicationScope, markedSourceEdge);
+ }
+ if( markedReversedEdge != null ){
+ indexService.queueDeleteEdge(applicationScope, markedReversedEdge);
+ }
- // item not deindexed, only edges
- removedEdges.forEach(edge -> {
- indexService.queueDeleteEdge(applicationScope, edge);
- });
// special handling for roles collection of a group
if ( headEntity.getType().equals( Group.ENTITY_TYPE ) ) {
@@ -572,7 +570,7 @@ public class CpRelationManager implements RelationManager {
if ( path.startsWith( "/roles/" ) ) {
Entity itemEntity =
- em.get( new SimpleEntityRef( memberEntity.getId().getType(), memberEntity.getId().getUuid() ) );
+ em.get( new SimpleEntityRef( entityId.getType(), entityId.getUuid() ) );
RoleRef roleRef = SimpleRoleRef.forRoleEntity( itemEntity );
em.deleteRole( roleRef.getApplicationRoleName(), Optional.fromNullable(itemEntity) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9c4b5242/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 02a7588..bbdce5a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -96,10 +96,9 @@ public class EventBuilderImpl implements EventBuilder {
logger.debug("Deleting in app scope {} with edge {}", applicationScope, edge);
}
- return indexService.deleteIndexEdge( applicationScope, edge ).flatMap( batch -> {
- final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- return gm.deleteEdge( edge ).map( deletedEdge -> batch );
- } );
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+ return gm.deleteEdge( edge )
+ .flatMap( deletedEdge -> indexService.deleteIndexEdge( applicationScope, deletedEdge ));
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9c4b5242/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 3b60b57..dc110f7 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -466,8 +466,8 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
SearchResponse searchResponse;
List<CandidateResult> candidates = new ArrayList<>();
- //never let the limit be less than 2 as there are potential indefinite paging issues
- final int searchLimit = Math.max(2, indexFig.getVersionQueryLimit());
+ // never let this fetch more than 100 to save memory
+ final int searchLimit = Math.min(100, indexFig.getVersionQueryLimit());
final QueryBuilder entityQuery = QueryBuilders
.termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(edge.getNodeId()));
@@ -485,41 +485,24 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
long queryTimestamp = 0L;
- while(true){
-
- QueryBuilder timestampQuery = QueryBuilders
- .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
- .gte(queryTimestamp);
-
- QueryBuilder finalQuery = QueryBuilders
- .boolQuery()
- .must(entityQuery)
- .must(timestampQuery);
-
- searchResponse = srb
- .setQuery(finalQuery)
- .setSize(searchLimit)
- .execute()
- .actionGet();
-
- int responseSize = searchResponse.getHits().getHits().length;
- if(responseSize == 0){
- break;
- }
- // update queryTimestamp to be the timestamp of the last entity returned from the query
- queryTimestamp = (long) searchResponse
- .getHits().getAt(responseSize - 1)
- .getSource().get(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME);
+ QueryBuilder timestampQuery = QueryBuilders
+ .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
+ .gte(queryTimestamp);
- candidates = aggregateScrollResults(candidates, searchResponse, null);
+ QueryBuilder finalQuery = QueryBuilders
+ .boolQuery()
+ .must(entityQuery)
+ .must(timestampQuery);
- if(responseSize < searchLimit){
+ searchResponse = srb
+ .setQuery(finalQuery)
+ .setSize(searchLimit)
+ .execute()
+ .actionGet();
- break;
- }
+ candidates = aggregateScrollResults(candidates, searchResponse, null);
- }
}
catch ( Throwable t ) {
logger.error( "Unable to communicate with Elasticsearch", t.getMessage() );
[15/50] usergrid git commit: Add edge read repair on entity updates.
Posted by mr...@apache.org.
Add edge read repair on entity updates.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a6c307b2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a6c307b2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a6c307b2
Branch: refs/heads/master
Commit: a6c307b282bbec2399f6dbf5220bb284184eddc3
Parents: f4ae0cb
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jun 29 15:50:49 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jun 29 15:50:49 2016 -0700
----------------------------------------------------------------------
.../usergrid/services/AbstractCollectionService.java | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6c307b2/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index 14b1df1..1ed7255 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -277,6 +277,12 @@ public class AbstractCollectionService extends AbstractService {
if ( item != null ) {
validateEntityType( item, id );
+
+ if( context.getOwner().getType().equals(TYPE_APPLICATION)) {
+ // this will repair any missing edges
+ em.addToCollection(context.getOwner(), context.getCollectionName(), item);
+ }
+
updateEntity( context, item, context.getPayload() );
item = importEntity( context, item );
}
@@ -312,7 +318,14 @@ public class AbstractCollectionService extends AbstractService {
else {
entity = importEntity( context, entity );
checkPermissionsForEntity( context, entity );
+
+ if( context.getOwner().getType().equals(TYPE_APPLICATION)) {
+ // this will repair any missing edges
+ em.addToCollection(context.getOwner(), context.getCollectionName(), entity);
+ }
+
updateEntity( context, entity );
+
}
return new ServiceResults( this, context, Type.COLLECTION, Results.fromEntity( entity ), null, null );
[04/50] usergrid git commit: Add application id to log for duplicate
unique values.
Posted by mr...@apache.org.
Add application id to log for duplicate unique values.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0c965ae2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0c965ae2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0c965ae2
Branch: refs/heads/master
Commit: 0c965ae221f9761f30e5c3fc5562b8e455a347bf
Parents: e982ba4
Author: Mike Dunker <md...@apigee.com>
Authored: Mon Jun 27 09:28:26 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Mon Jun 27 09:28:26 2016 -0700
----------------------------------------------------------------------
.../impl/UniqueValueSerializationStrategyImpl.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0c965ae2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index 1c5b08f..28e28aa 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -333,8 +333,9 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
try {
- logger.warn("Duplicate unique value [{}={}] found, removing newer entry " +
- "with entity id [{}] and entity version [{}]", field.getName(), field.getValue().toString(),
+ logger.warn("Duplicate unique value [{}={}] found for application [{}], removing newer " +
+ "entry with entity id [{}] and entity version [{}]", field.getName(),
+ field.getValue().toString(), applicationId.getUuid(),
candidate.getEntityId().getUuid(), candidate.getEntityVersion() );
delete(appScope, candidate ).execute();
[13/50] usergrid git commit: Merge branch 'release-2.1.1' of
https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/426449e3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/426449e3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/426449e3
Branch: refs/heads/master
Commit: 426449e3a2ce2365d4ebf1e0ed498b75cf16d2f6
Parents: 5803916 37c482c
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jun 29 10:53:49 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jun 29 10:53:49 2016 -0700
----------------------------------------------------------------------
docs/orgs-and-apps/application.md | 104 +++++++++++++++++++++++++++++++++
1 file changed, 104 insertions(+)
----------------------------------------------------------------------
[08/50] usergrid git commit: Improve read repair for missing Edge
(app->collection). Also enhance read repair for unique value cleanup to
happen only upon initial read before write-first strategy for Unique Values.
Add a bunch more tests around the read
Posted by mr...@apache.org.
Improve read repair for missing Edge (app->collection). Also enhance read repair for unique value cleanup to happen only upon initial read before write-first strategy for Unique Values. Add a bunch more tests around the read repairs.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/724968a2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/724968a2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/724968a2
Branch: refs/heads/master
Commit: 724968a2bc354e3c3f317e1d0b98026c2fe3baeb
Parents: 9c4b524
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jun 29 01:45:57 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jun 29 01:45:57 2016 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 6 ++
.../corepersistence/CpRelationManager.java | 42 +++++++++
.../usergrid/persistence/EntityManager.java | 16 +++-
.../usergrid/persistence/RelationManager.java | 2 +
.../impl/EntityCollectionManagerImpl.java | 5 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 13 ++-
.../UniqueValueSerializationStrategy.java | 7 +-
.../UniqueValueSerializationStrategyImpl.java | 89 +++++++++++---------
...iqueValueSerializationStrategyProxyImpl.java | 8 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 71 ++++++++++++++++
...niqueValueSerializationStrategyImplTest.java | 26 ++++--
.../collection/users/PermissionsResourceIT.java | 4 +-
.../services/AbstractCollectionService.java | 37 ++++++--
.../usergrid/services/ServiceInvocationIT.java | 28 +++++-
14 files changed, 280 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index ab62b36..3dc0d13 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -1545,6 +1545,12 @@ public class CpEntityManager implements EntityManager {
getRelationManager( entityRef ).removeFromCollection( collectionName, itemRef );
}
+ @Override
+ public void removeItemFromCollection( EntityRef entityRef, String collectionName, EntityRef itemRef ) throws Exception {
+
+ getRelationManager( entityRef ).removeItemFromCollection( collectionName, itemRef );
+ }
+
@Override
public Set<String> getCollectionIndexes( EntityRef entity, String collectionName ) throws Exception {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 6e1bade..fbf0b14 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -579,6 +579,48 @@ public class CpRelationManager implements RelationManager {
}
}
+ @Override
+ public void removeItemFromCollection( String collectionName, EntityRef itemRef ) throws Exception {
+
+ Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
+
+ // remove edge from collection to item
+ GraphManager gm = managerCache.getGraphManager( applicationScope );
+
+
+
+ // mark the edge versions and take the first for later delete edge queue event ( load is descending )
+ final Edge markedSourceEdge = gm.loadEdgeVersions(
+ CpNamingUtils.createEdgeFromCollectionName( cpHeadEntity.getId(), collectionName, entityId ) )
+ .flatMap(edge -> gm.markEdge(edge)).toBlocking().firstOrDefault(null);
+
+
+ Edge markedReversedEdge = null;
+ CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName );
+ if (collection != null && collection.getLinkedCollection() != null) {
+ // delete reverse edges
+ final String pluralType = InflectionUtils.pluralize( cpHeadEntity.getId().getType() );
+ markedReversedEdge = gm.loadEdgeVersions(
+ CpNamingUtils.createEdgeFromCollectionName( entityId, pluralType, cpHeadEntity.getId() ) )
+ .flatMap(reverseEdge -> gm.markEdge(reverseEdge)).toBlocking().firstOrDefault(null);
+ }
+
+
+ /**
+ * Remove from the index. This will call gm.deleteEdge which also deletes the reverse edge(s) and de-indexes
+ * older versions of the edge(s).
+ *
+ */
+ if( markedSourceEdge != null ) {
+ indexService.queueDeleteEdge(applicationScope, markedSourceEdge);
+ }
+ if( markedReversedEdge != null ){
+ indexService.queueDeleteEdge(applicationScope, markedReversedEdge);
+
+ }
+
+ }
+
@Override
public void copyRelationships( String srcRelationName, EntityRef dstEntityRef, String dstRelationName )
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 53a7a89..7e25a80 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -394,17 +394,29 @@ public interface EntityManager {
Map<String, Object> properties ) throws Exception;
/**
- * Removes an entity to the specified collection belonging to the specified entity.
+ * Deletes an entity from the specified collection.
*
* @param entityRef an entity reference
* @param collectionName the collection name.
- * @param itemRef a entity to be removed from the collection.
+ * @param itemRef a entity to be deleted and removed from the collection.
*
* @throws Exception the exception
*/
public void removeFromCollection( EntityRef entityRef, String collectionName, EntityRef itemRef)
throws Exception;
+ /**
+ * Removes only the edge from the specified collection, the entity is left in-tact
+ *
+ * @param entityRef an entity reference
+ * @param collectionName the collection name.
+ * @param itemRef a entity to be removed from the collection.
+ *
+ * @throws Exception the exception
+ */
+ public void removeItemFromCollection( EntityRef entityRef, String collectionName, EntityRef itemRef)
+ throws Exception;
+
public Results searchCollection( EntityRef entityRef, String collectionName, Query query )
throws Exception;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
index 0011183..f0647ac 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/RelationManager.java
@@ -59,6 +59,8 @@ public interface RelationManager {
public void removeFromCollection( String collectionName, EntityRef itemRef ) throws Exception;
+ public void removeItemFromCollection( String collectionName, EntityRef itemRef ) throws Exception;
+
public void copyRelationships( String srcRelationName, EntityRef dstEntityRef, String dstRelationName )
throws Exception;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 1ccc18f..523b4df 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -385,10 +385,11 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
if ( entity == null || !entity.getEntity().isPresent() ) {
if(logger.isTraceEnabled()) {
- logger.trace("Unique value [{}={}] does not have corresponding entity, executing " +
+ logger.trace("Unique value [{}={}] does not have corresponding entity [{}], executing " +
"read repair to remove stale unique value entry",
expectedUnique.getField().getName(),
- expectedUnique.getField().getValue().toString()
+ expectedUnique.getField().getValue().toString(),
+ expectedUnique.getEntityId()
);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 7b76dc8..d7c8ecd 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -122,7 +122,9 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
try {
// loading will retrieve the oldest unique value entry for the field
- UniqueValueSet set = uniqueValueStrat.load(scope, written.getEntityId().getType(), Collections.singletonList(written.getField()));
+ // purposely enable the read repair here to clean up before we write
+ UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getReadCL(),
+ written.getEntityId().getType(), Collections.singletonList(written.getField()), true);
set.forEach(uniqueValue -> {
@@ -149,6 +151,11 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
}
if(preWriteUniquenessViolations.size() > 0 ){
+ if(logger.isTraceEnabled()){
+ logger.trace("Pre-write unique violations found, raising exception before executing first write");
+ }
+ logger.error("Pre-write unique violations found, raising exception before executing first write");
+
throw new WriteUniqueVerifyException(mvccEntity, scope,
preWriteUniquenessViolations );
}
@@ -217,7 +224,9 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
final UniqueValueSet uniqueValues;
try {
// load ascending for verification to make sure we wrote is the last read back
- uniqueValues = uniqueValueSerializationStrategy.load( scope, consistencyLevel, type, uniqueFields );
+ // don't read repair on this read because our write-first strategy will introduce a duplicate
+ uniqueValues =
+ uniqueValueSerializationStrategy.load( scope, consistencyLevel, type, uniqueFields, false);
}
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to read from cassandra", e );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
index 95cfa68..35bb1b8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
@@ -79,11 +79,12 @@ public interface UniqueValueSerializationStrategy extends Migration, VersionedDa
* @param consistencyLevel Consistency level of query
* @param type The type the unique value exists within
* @param fields Field name/value to search for
- * @return UniqueValueSet containing fields from the collection that exist in cassandra
+ * @param useReadRepair
+ * @return UniqueValueSet containing fields from the collection that exist in cassandra
* @throws ConnectionException on error connecting to Cassandra
*/
- UniqueValueSet load( ApplicationScope applicationScope, ConsistencyLevel consistencyLevel, String type,
- Collection<Field> fields ) throws ConnectionException;
+ UniqueValueSet load(ApplicationScope applicationScope, ConsistencyLevel consistencyLevel, String type,
+ Collection<Field> fields, boolean useReadRepair) throws ConnectionException;
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index aec2e58..db93272 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -237,13 +237,13 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
@Override
public UniqueValueSet load( final ApplicationScope colScope, final String type, final Collection<Field> fields )
throws ConnectionException {
- return load( colScope, ConsistencyLevel.valueOf( cassandraFig.getReadCL() ), type, fields );
+ return load( colScope, ConsistencyLevel.valueOf( cassandraFig.getReadCL() ), type, fields, false);
}
@Override
- public UniqueValueSet load( final ApplicationScope appScope, final ConsistencyLevel consistencyLevel,
- final String type, final Collection<Field> fields ) throws ConnectionException {
+ public UniqueValueSet load(final ApplicationScope appScope, final ConsistencyLevel consistencyLevel,
+ final String type, final Collection<Field> fields, boolean useReadRepair) throws ConnectionException {
Preconditions.checkNotNull( fields, "fields are required" );
Preconditions.checkArgument( fields.size() > 0, "More than 1 field must be specified" );
@@ -307,71 +307,78 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
final UniqueValue uniqueValue =
new UniqueValueImpl(field, entityVersion.getEntityId(), entityVersion.getEntityVersion());
-
// set the initial candidate and move on
- if(candidates.size() == 0){
+ if (candidates.size() == 0) {
candidates.add(uniqueValue);
continue;
}
- final int result = uniqueValueComparator.compare(uniqueValue, candidates.get(candidates.size() -1));
+ if(!useReadRepair){
- if(result == 0){
+ // take only the first
+ break;
- // do nothing, only versions can be newer and we're not worried about newer versions of same entity
- if(logger.isTraceEnabled()){
- logger.trace("Candidate unique value is equal to the current unique value");
- }
+ } else {
- // update candidate w/ latest version
- candidates.add(uniqueValue);
- }else if(result < 0){
+ final int result = uniqueValueComparator.compare(uniqueValue, candidates.get(candidates.size() - 1));
- // delete the duplicate from the unique value index
- candidates.forEach(candidate -> {
+ if (result == 0) {
- try {
+ // do nothing, only versions can be newer and we're not worried about newer versions of same entity
+ if (logger.isTraceEnabled()) {
+ logger.trace("Candidate unique value is equal to the current unique value");
+ }
- logger.warn("Duplicate unique value [{}={}] found for application [{}], removing newer " +
- "entry with entity id [{}] and entity version [{}]", field.getName(),
- field.getValue().toString(), applicationId.getUuid(),
- candidate.getEntityId().getUuid(), candidate.getEntityVersion() );
+ // update candidate w/ latest version
+ candidates.add(uniqueValue);
- delete(appScope, candidate ).execute();
+ } else if (result < 0) {
- } catch (ConnectionException e) {
- // do nothing for now
- }
+ // delete the duplicate from the unique value index
+ candidates.forEach(candidate -> {
- });
+ try {
- // clear the transient candidates list
- candidates.clear();
+ logger.warn("Duplicate unique value [{}={}] found for application [{}], removing newer " +
+ "entry with entity id [{}] and entity version [{}]", field.getName(),
+ field.getValue().toString(), applicationId.getUuid(),
+ candidate.getEntityId().getUuid(), candidate.getEntityVersion());
- if(logger.isTraceEnabled()) {
- logger.trace("Updating candidate to entity id [{}] and entity version [{}]",
- uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion());
+ delete(appScope, candidate).execute();
- }
+ } catch (ConnectionException e) {
+ // do nothing for now
+ }
- // add our new candidate to the list
- candidates.add(uniqueValue);
+ });
+
+ // clear the transient candidates list
+ candidates.clear();
+ if (logger.isTraceEnabled()) {
+ logger.trace("Updating candidate to entity id [{}] and entity version [{}]",
+ uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion());
- }else{
+ }
- logger.warn("Duplicate unique value [{}={}] found for application [{}], removing newer entry " +
- "with entity id [{}] and entity version [{}].", field.getName(), field.getValue().toString(),
- applicationId.getUuid(), uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion() );
+ // add our new candidate to the list
+ candidates.add(uniqueValue);
- // delete the duplicate from the unique value index
- delete(appScope, uniqueValue ).execute();
+ } else {
- }
+ logger.warn("Duplicate unique value [{}={}] found for application [{}], removing newer entry " +
+ "with entity id [{}] and entity version [{}].", field.getName(), field.getValue().toString(),
+ applicationId.getUuid(), uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion());
+ // delete the duplicate from the unique value index
+ delete(appScope, uniqueValue).execute();
+
+ }
+
+ }
}
// take the last candidate ( should be the latest version) and add to the result set
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java
index 1de4052..b9c9999 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java
@@ -115,16 +115,16 @@ public class UniqueValueSerializationStrategyProxyImpl implements UniqueValueSer
@Override
- public UniqueValueSet load( final ApplicationScope applicationScope, final ConsistencyLevel consistencyLevel,
- final String type, final Collection<Field> fields ) throws ConnectionException {
+ public UniqueValueSet load(final ApplicationScope applicationScope, final ConsistencyLevel consistencyLevel,
+ final String type, final Collection<Field> fields, boolean useReadRepair) throws ConnectionException {
final MigrationRelationship<UniqueValueSerializationStrategy> migration = getMigrationRelationShip();
if ( migration.needsMigration() ) {
- return migration.from.load( applicationScope, type, fields );
+ return migration.from.load( applicationScope, consistencyLevel, type, fields, useReadRepair );
}
- return migration.to.load( applicationScope, type, fields );
+ return migration.to.load( applicationScope, consistencyLevel, type, fields, useReadRepair );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
index 9d0cd20..3d411a4 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
@@ -18,6 +18,12 @@
package org.apache.usergrid.persistence.collection.mvcc.stage.write;
+import org.apache.usergrid.persistence.collection.FieldSet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -41,6 +47,8 @@ import org.apache.usergrid.persistence.model.field.StringField;
import com.google.inject.Inject;
+import java.util.Collections;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -60,6 +68,9 @@ public class WriteUniqueVerifyIT {
public MigrationManagerRule migrationManagerRule;
@Inject
+ public UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+
+ @Inject
public EntityCollectionManagerFactory cmf;
@Test
@@ -142,4 +153,64 @@ public class WriteUniqueVerifyIT {
entity.setField( new StringField("foo", "bar"));
entityManager.write( entity ).toBlocking().last();
}
+
+ @Test
+ public void testConflictReadRepair() throws Exception {
+
+ final Id appId = new SimpleId("testNoConflict");
+
+
+
+ final ApplicationScope scope = new ApplicationScopeImpl( appId);
+
+ final EntityCollectionManager entityManager = cmf.createCollectionManager( scope );
+
+ final Entity entity = TestEntityGenerator.generateEntity();
+ entity.setField(new StringField("name", "Porsche 911 GT3", true));
+ entity.setField(new StringField("identifier", "911gt3", true));
+ entity.setField(new IntegerField("top_speed_mph", 194));
+ entityManager.write( entity ).toBlocking().last();
+
+
+ FieldSet fieldSet =
+ entityManager.getEntitiesFromFields("test", Collections.singletonList(entity.getField("name")), true)
+ .toBlocking().last();
+
+ MvccEntity entityFetched = fieldSet.getEntity( entity.getField("name") );
+
+
+ final Entity entityDuplicate = TestEntityGenerator.generateEntity();
+ UniqueValue uniqueValue = new UniqueValueImpl(new StringField("name", "Porsche 911 GT3", true),
+ entityDuplicate.getId(), UUIDGenerator.newTimeUUID());
+
+ // manually insert a record to simulate a 'duplicate' trying to be inserted
+ uniqueValueSerializationStrategy.
+ write(scope, uniqueValue).execute();
+
+
+
+ FieldSet fieldSetAgain =
+ entityManager.getEntitiesFromFields("test", Collections.singletonList(entity.getField("name")), true)
+ .toBlocking().last();
+
+ MvccEntity entityFetchedAgain = fieldSetAgain.getEntity( entity.getField("name") );
+
+ assertEquals(entityFetched, entityFetchedAgain);
+
+
+ // now test writing the original entity again ( simulates a PUT )
+ // this should read repair and work
+ entityManager.write( entity ).toBlocking().last();
+
+ FieldSet fieldSetAgainAgain =
+ entityManager.getEntitiesFromFields("test", Collections.singletonList(entity.getField("name")), true)
+ .toBlocking().last();
+
+ MvccEntity entityFetchedAgainAgain = fieldSetAgainAgain.getEntity( entity.getField("name") );
+
+ assertEquals(entityFetched, entityFetchedAgainAgain);
+
+
+
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java
index ed3e42b..3dbf1ec 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImplTest.java
@@ -23,6 +23,7 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.UUID;
+import com.netflix.astyanax.model.ConsistencyLevel;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
@@ -369,7 +370,8 @@ public abstract class UniqueValueSerializationStrategyImplTest {
strategy.write( scope, stored2 ).execute();
// load descending to get the older version of entity for this unique value
- UniqueValueSet fields = strategy.load( scope, entityId1.getType(), Collections.<Field>singleton( field ));
+ UniqueValueSet fields = strategy.load( scope, ConsistencyLevel.CL_LOCAL_QUORUM,
+ entityId1.getType(), Collections.<Field>singleton( field ), true);
UniqueValue retrieved = fields.getValue( field.getName() );
@@ -383,7 +385,8 @@ public abstract class UniqueValueSerializationStrategyImplTest {
strategy.write( scope, stored3 ).execute();
// load the values again, we should still only get back the original unique value
- fields = strategy.load( scope, entityId1.getType(), Collections.<Field>singleton( field ));
+ fields = strategy.load( scope, ConsistencyLevel.CL_LOCAL_QUORUM,
+ entityId1.getType(), Collections.<Field>singleton( field ), true);
retrieved = fields.getValue( field.getName() );
@@ -396,7 +399,8 @@ public abstract class UniqueValueSerializationStrategyImplTest {
strategy.write( scope, stored4 ).execute();
// load the values again, now we should get the latest version of the original UUID written
- fields = strategy.load( scope, entityId1.getType(), Collections.<Field>singleton( field ));
+ fields = strategy.load( scope, ConsistencyLevel.CL_LOCAL_QUORUM,
+ entityId1.getType(), Collections.<Field>singleton( field ), true);
retrieved = fields.getValue( field.getName() );
@@ -433,7 +437,8 @@ public abstract class UniqueValueSerializationStrategyImplTest {
strategy.write( scope, stored2 ).execute();
// load descending to get the older version of entity for this unique value
- UniqueValueSet fields = strategy.load( scope, entityId1.getType(), Collections.<Field>singleton( field ));
+ UniqueValueSet fields = strategy.load( scope, ConsistencyLevel.CL_LOCAL_QUORUM,
+ entityId1.getType(), Collections.<Field>singleton( field ), true);
UniqueValue retrieved = fields.getValue( field.getName() );
Assert.assertNotNull( retrieved );
@@ -470,7 +475,12 @@ public abstract class UniqueValueSerializationStrategyImplTest {
// load descending to get the older version of entity for this unique value
- UniqueValueSet fields = strategy.load( scope, entityId1.getType(), Collections.<Field>singleton( field ));
+ UniqueValueSet fields = strategy.load( scope, ConsistencyLevel.CL_LOCAL_QUORUM,
+ entityId1.getType(), Collections.<Field>singleton( field ), true);
+
+
+ fields = strategy.load( scope, ConsistencyLevel.CL_LOCAL_QUORUM,
+ entityId1.getType(), Collections.<Field>singleton( field ), false);
UniqueValue retrieved = fields.getValue( field.getName() );
assertEquals( stored3, retrieved );
@@ -506,7 +516,8 @@ public abstract class UniqueValueSerializationStrategyImplTest {
// load descending to get the older version of entity for this unique value
- UniqueValueSet fields = strategy.load( scope, entityId1.getType(), Collections.<Field>singleton( field ));
+ UniqueValueSet fields = strategy.load( scope,
+ ConsistencyLevel.CL_LOCAL_QUORUM, entityId1.getType(), Collections.<Field>singleton( field ), true);
UniqueValue retrieved = fields.getValue( field.getName() );
assertEquals( stored1, retrieved );
@@ -549,7 +560,8 @@ public abstract class UniqueValueSerializationStrategyImplTest {
// load descending to get the older version of entity for this unique value
- UniqueValueSet fields = strategy.load( scope, entityId1.getType(), Collections.<Field>singleton( field ));
+ UniqueValueSet fields = strategy.load( scope, ConsistencyLevel.CL_LOCAL_QUORUM,
+ entityId1.getType(), Collections.<Field>singleton( field ), true);
UniqueValue retrieved = fields.getValue( field.getName() );
assertEquals( stored1, retrieved );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java
index b0b3791..5380e00 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/PermissionsResourceIT.java
@@ -116,7 +116,7 @@ public class PermissionsResourceIT extends AbstractRestIT {
}
// check if the role was assigned
- assertEquals(status, 404);
+ assertEquals(404, status);
}
@@ -167,7 +167,7 @@ public class PermissionsResourceIT extends AbstractRestIT {
fail("Should not have been able to retrieve the user as it was deleted");
}catch (ClientErrorException e){
status=e.getResponse().getStatus();
- assertEquals( 404,status );
+ assertEquals( 404, status );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index 7539e0c..14b1df1 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -41,6 +41,7 @@ import org.apache.usergrid.services.ServiceResults.Type;
import org.apache.usergrid.services.exceptions.ForbiddenServiceOperationException;
import org.apache.usergrid.services.exceptions.ServiceResourceNotFoundException;
+import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
import static org.apache.usergrid.utils.ClassUtils.cast;
@@ -125,16 +126,34 @@ public class AbstractCollectionService extends AbstractService {
checkPermissionsForEntity( context, entity );
}
- // the context of the entity they're trying to load isn't owned by the owner
- // in the path, don't return it
+ // check ownership based on graph
if ( !em.isCollectionMember( context.getOwner(), context.getCollectionName(), entity ) ) {
- logger.info( "Someone tried to GET entity {} they don't own. Entity id {} with owner {}",
- getEntityType(), id, context.getOwner()
- );
- throw new ServiceResourceNotFoundException( context );
- }
- // TODO check that entity is in fact in the collection
+ // the entity is already loaded in the scope of the owner and type ( collection ) so it must exist at this point
+ // if for some reason it's not a member of the collection, it should be and read repair it
+ if( context.getOwner().getType().equals(TYPE_APPLICATION) ){
+ logger.warn( "Edge missing for entity id {} with owner {}. Executing edge read repair to create new edge in " +
+ "collection {}", id, context.getOwner(), context.getCollectionName());
+
+ em.addToCollection( context.getOwner(), context.getCollectionName(), entity);
+
+ // do a final check to be absolutely sure we're good now before returning back to the client
+ // TODO : Keep thinking if the double-check read after repair is necessary. Favoring stability here
+ if ( !em.isCollectionMember( context.getOwner(), context.getCollectionName(), entity ) ) {
+ logger.error( "Edge read repair failed for entity id {} with owner {} in collection {}",
+ id, context.getOwner(), context.getCollectionName());
+
+ throw new ServiceResourceNotFoundException( context );
+ }
+
+ }
+ // if not head application, then we can't assume the ownership is meant to be there
+ else{
+ throw new ServiceResourceNotFoundException( context );
+ }
+
+
+ }
List<ServiceRequest> nextRequests = context.getNextServiceRequests( entity );
@@ -158,7 +177,7 @@ public class AbstractCollectionService extends AbstractService {
if ( entityId == null ) {
if (logger.isTraceEnabled()) {
- logger.trace("miss on entityType: {} with name: {}", getEntityType(), name);
+ logger.trace("Miss on entityType: {} with name: {}", getEntityType(), name);
}
String msg = "Cannot find entity with name: "+name;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/724968a2/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
index 57f0bb2..8c2be2c 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
@@ -24,11 +24,14 @@ import java.util.Map;
import java.util.UUID;
import org.apache.usergrid.cassandra.ClearShiroSubject;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import org.apache.usergrid.utils.InflectionUtils;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
@@ -221,4 +224,25 @@ public class ServiceInvocationIT extends AbstractServiceIT {
app.testRequest( ServiceAction.GET, 1,
"projects", project.getName(), "contains", "contributors", contributor.getName());
}
+
+ @Test
+ public void testGetByIdAndNameEdgeReadRepair() throws Exception {
+
+ EntityManager em = setup.getEmf().getEntityManager( app.getId() );
+
+ Entity contributor = app.doCreate( "contributor", "Malaka" );
+
+ EntityRef appRef = new SimpleEntityRef("application", app.getId());
+
+
+ em.removeItemFromCollection(appRef, InflectionUtils.pluralize(contributor.getType()), contributor);
+
+ assertFalse("Entity should not have an edge from app to entity",
+ em.isCollectionMember(appRef, InflectionUtils.pluralize(contributor.getType()), contributor));
+
+ app.testRequest( ServiceAction.GET, 1, "contributor", contributor.getName());
+
+ assertTrue("Entity should now be member of the collection",
+ em.isCollectionMember(appRef, InflectionUtils.pluralize(contributor.getType()), contributor));
+ }
}
[50/50] usergrid git commit: Merge branch 'release-2.1.1'
Posted by mr...@apache.org.
Merge branch 'release-2.1.1'
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/849df674
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/849df674
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/849df674
Branch: refs/heads/master
Commit: 849df67462ecb78b9fd1690c93e969fefc407c51
Parents: b0fbf14 a8f4218
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jul 13 22:37:46 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jul 13 22:37:46 2016 -0700
----------------------------------------------------------------------
docs/data-storage/collections.md | 144 ++++
docs/orgs-and-apps/application.md | 104 +++
.../main/resources/usergrid-default.properties | 44 +-
.../src/test/resources/usergrid-test.properties | 20 +-
stack/core/pom.xml | 819 ++++++++++---------
.../usergrid/corepersistence/CoreModule.java | 66 +-
.../corepersistence/CpEntityManager.java | 378 +++++----
.../corepersistence/CpEntityManagerFactory.java | 68 +-
.../corepersistence/CpRelationManager.java | 99 ++-
.../corepersistence/EntityManagerFig.java | 9 +
.../asyncevents/EventBuilderImpl.java | 7 +-
.../index/CollectionSettingsCache.java | 47 ++
.../index/CollectionSettingsCacheFactory.java | 44 +
.../index/CollectionSettingsCacheFig.java | 39 +
.../index/CollectionSettingsCacheImpl.java | 122 +++
.../corepersistence/index/IndexSchemaCache.java | 50 --
.../index/IndexSchemaCacheFactory.java | 44 -
.../index/IndexSchemaCacheFig.java | 39 -
.../index/IndexSchemaCacheImpl.java | 120 ---
.../corepersistence/index/IndexServiceImpl.java | 65 +-
.../index/ReIndexServiceImpl.java | 44 +-
.../service/ApplicationServiceImpl.java | 22 +-
.../usergrid/persistence/EntityManager.java | 26 +-
.../apache/usergrid/persistence/PathQuery.java | 3 +-
.../usergrid/persistence/RelationManager.java | 2 +
.../index/AsyncIndexServiceTest.java | 2 +-
.../corepersistence/index/IndexServiceTest.java | 92 ++-
.../resources/usergrid-custom-test.properties | 13 +
stack/corepersistence/actorsystem/pom.xml | 106 +++
.../persistence/actorsystem/ActorSystemFig.java | 83 ++
.../actorsystem/ActorSystemManager.java | 86 ++
.../actorsystem/ActorSystemManagerImpl.java | 459 +++++++++++
.../actorsystem/ActorSystemModule.java | 34 +
.../persistence/actorsystem/ClientActor.java | 205 +++++
.../actorsystem/GuiceActorProducer.java | 46 ++
.../persistence/actorsystem/RouterProducer.java | 51 ++
.../src/main/resources/application.conf | 50 ++
.../actorsystem/ActorServiceServiceTest.java | 74 ++
stack/corepersistence/collection/pom.xml | 120 ++-
.../collection/EntityCollectionManager.java | 8 +-
.../EntityCollectionManagerFactory.java | 4 +-
.../persistence/collection/FieldSet.java | 12 +
.../exception/WriteUniqueVerifyException.java | 2 +-
.../collection/guice/CollectionModule.java | 9 +
.../EntityCollectionManagerFactoryImpl.java | 166 ++--
.../impl/EntityCollectionManagerImpl.java | 180 ++--
.../mvcc/stage/CollectionIoEvent.java | 14 +-
.../mvcc/stage/delete/MarkCommit.java | 36 +-
.../mvcc/stage/write/WriteCommit.java | 82 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 142 +++-
.../serialization/SerializationFig.java | 4 -
.../UniqueValueSerializationStrategy.java | 25 +-
.../serialization/impl/MutableFieldSet.java | 12 +
.../impl/UniqueFieldRowKeySerializer.java | 3 +-
.../UniqueValueSerializationStrategyImpl.java | 138 +++-
...iqueValueSerializationStrategyProxyImpl.java | 21 +-
.../uniquevalues/ReservationCache.java | 87 ++
.../uniquevalues/ReservationCacheActor.java | 87 ++
.../uniquevalues/UniqueValueActor.java | 282 +++++++
.../uniquevalues/UniqueValueException.java | 33 +
.../uniquevalues/UniqueValuesFig.java | 67 ++
.../uniquevalues/UniqueValuesRouter.java | 70 ++
.../uniquevalues/UniqueValuesService.java | 74 ++
.../uniquevalues/UniqueValuesServiceImpl.java | 372 +++++++++
.../uniquevalues/UniqueValuesTable.java | 42 +
.../uniquevalues/UniqueValuesTableImpl.java | 94 +++
.../collection/AbstractUniqueValueTest.java | 50 ++
.../collection/EntityCollectionManagerIT.java | 171 ++--
.../EntityCollectionManagerStressTest.java | 21 +-
.../collection/guice/TestCollectionModule.java | 17 +
.../mvcc/stage/delete/MarkCommitTest.java | 10 +-
.../mvcc/stage/write/WriteCommitTest.java | 8 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 121 ++-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 55 +-
...niqueValueSerializationStrategyImplTest.java | 26 +-
.../UniqueValuesServiceDeleteTest.java | 149 ++++
.../uniquevalues/UniqueValuesServiceTest.java | 185 +++++
.../src/test/resources/usergrid-CHOP.properties | 18 +
.../src/test/resources/usergrid-UNIT.properties | 19 +
.../src/test/resources/usergrid.properties | 32 +
stack/corepersistence/common/pom.xml | 23 +-
.../core/guice/SettingsValidationCluster.java | 18 +
.../core/guice/MigrationManagerRule.java | 18 +
.../guice/SettingsValidationClusterTest.java | 19 +
.../src/test/resources/usergrid-UNIT.properties | 19 +
.../src/test/resources/usergrid.properties | 19 +
stack/corepersistence/model/pom.xml | 16 +
.../persistence/model/entity/EntityMap.java | 18 +
.../persistence/model/field/ArrayField.java | 7 +
.../persistence/model/field/ByteArrayField.java | 9 +
.../persistence/model/field/DistanceField.java | 9 +
.../model/field/EntityObjectField.java | 9 +
.../persistence/model/field/FieldTypeName.java | 3 -
.../persistence/model/field/ListField.java | 7 +
.../persistence/model/field/LocationField.java | 7 +
.../persistence/model/field/NullField.java | 7 +
.../persistence/model/field/SetField.java | 7 +
.../model/field/value/EntityObject.java | 2 +-
.../persistence/model/util/EntityUtils.java | 18 +
.../persistence/model/util/UUIDGenerator.java | 18 +
.../persistence/model/field/EntityTest.java | 18 +
.../model/util/UUIDGeneratorTest.java | 18 +
stack/corepersistence/pom.xml | 72 ++
stack/corepersistence/queryindex/pom.xml | 18 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 2 +
.../index/impl/EsEntityIndexImpl.java | 53 +-
stack/pom.xml | 6 -
.../usergrid/rest/AbstractContextResource.java | 1 +
.../apache/usergrid/rest/ShutdownListener.java | 11 +
.../rest/applications/CollectionResource.java | 97 +--
.../rest/applications/ServiceResource.java | 8 +-
...ApplicationAlreadyExistsExceptionMapper.java | 32 +
.../apache/usergrid/rest/NotificationsIT.java | 6 +-
.../org/apache/usergrid/rest/UniqueCatsIT.java | 233 ++++++
.../apache/usergrid/rest/UniqueValuesIT.java | 224 +++++
.../rest/UniqueValuesPerformanceIT.java | 165 ++++
.../rest/applications/ApplicationDeleteIT.java | 115 ++-
.../applications/ApplicationResourceIT.java | 2 +-
.../collection/CollectionsResourceIT.java | 251 +++---
.../collection/users/PermissionsResourceIT.java | 4 +-
.../resources/corepersistence-UNIT.properties | 2 +
.../resources/usergrid-custom-test.properties | 19 +-
.../resources/usergrid-rest-deploy-context.xml | 33 +-
stack/services/pom.xml | 6 +
.../services/AbstractCollectionService.java | 71 +-
.../services/AbstractConnectionsService.java | 9 +-
.../usergrid/services/AbstractService.java | 53 +-
.../applications/ApplicationsService.java | 4 +-
.../resources/usergrid-services-context.xml | 2 +-
.../usergrid/services/ServiceInvocationIT.java | 28 +-
.../usergrid/tools/UniqueValueScanner.java | 124 +--
131 files changed, 6850 insertions(+), 1884 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/849df674/stack/core/pom.xml
----------------------------------------------------------------------
diff --cc stack/core/pom.xml
index ce5b67d,9c6497e..6c70342
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@@ -17,30 -17,30 +17,31 @@@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>usergrid</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- <relativePath>../</relativePath>
- </parent>
-
- <artifactId>usergrid-core</artifactId>
- <name>Usergrid Core</name>
- <description>Core services for Usergrid system.</description>
- <packaging>jar</packaging>
-
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- </plugin>
- </plugins>
- </reporting>
-
-
- <build>
++
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid</artifactId>
- <version>2.1.1-SNAPSHOT</version>
++ <version>2.2.0-SNAPSHOT</version>
+ <relativePath>../</relativePath>
+ </parent>
+
+ <artifactId>usergrid-core</artifactId>
+ <name>Usergrid Core</name>
+ <description>Core services for Usergrid system.</description>
+ <packaging>jar</packaging>
+
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ </plugin>
+ </plugins>
+ </reporting>
+
+
+ <build>
<resources>
<resource>
@@@ -55,384 -55,395 +56,396 @@@
</includes>
</resource>
</resources>
- <testResources>
- <testResource>
- <directory>src/test/resources</directory>
- <filtering>true</filtering>
- <includes>
- <include>**/*.yaml</include>
- <include>**/*.properties</include>
- <include>**/*.xml</include>
- </includes>
- </testResource>
- <testResource>
- <directory>src/test/resources</directory>
- <filtering>true</filtering>
- <includes>
- <include>largeentity.json</include>
- </includes>
- </testResource>
- </testResources>
-
- <plugins>
-
- <!--
- Do not need to configure surefire plugin here, parent POM configuration is sufficient.
- -->
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <id>test-jar-execution</id>
- <phase>package</phase>
- <goals>
- <goal>test-jar</goal>
- </goals>
- <configuration>
- <includes>
- <include>**/org/apache/usergrid/**</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- </plugins>
- </build>
-
- <dependencies>
-
-
- <!-- dependent on wiring guice and spring -->
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-spring</artifactId>
- <version>4.0-beta5</version>
- </dependency>
-
- <!-- Apache Dependencies -->
-
- <dependency>
- <groupId>org.apache.cassandra</groupId>
- <artifactId>cassandra-all</artifactId>
- <!-- Exclude the old and problematic Snappy -->
- <exclusions>
- <exclusion>
- <artifactId>snappy-java</artifactId>
- <groupId>org.xerial.snappy</groupId>
- </exclusion>
- <exclusion>
- <artifactId>antlr</artifactId>
- <groupId>org.antlr</groupId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.cassandra</groupId>
- <artifactId>cassandra-thrift</artifactId>
- </dependency>
-
- <!-- Include the slighly newer and less problematic snappy -->
- <dependency>
- <artifactId>snappy-java</artifactId>
- <groupId>org.xerial.snappy</groupId>
- <version>1.0.5</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-collections</groupId>
- <artifactId>commons-collections</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </dependency>
-
- <dependency>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- </dependency>
-
- <!-- SUN, Javax Package, and Other Commercial Dependencies -->
-
- <dependency>
- <groupId>com.beust</groupId>
- <artifactId>jcommander</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.sun.mail</groupId>
- <artifactId>javax.mail</artifactId>
- </dependency>
-
- <dependency>
- <groupId>javax.activation</groupId>
- <artifactId>activation</artifactId>
- </dependency>
-
- <dependency>
- <groupId>javax.persistence</groupId>
- <artifactId>persistence-api</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.fasterxml.uuid</groupId>
- <artifactId>java-uuid-generator</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.hazelcast</groupId>
- <artifactId>hazelcast-all</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.netflix.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.netflix.astyanax</groupId>
- <artifactId>astyanax-recipes</artifactId>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
-
- <!-- Codehaus, Spring and Other Org Dependencies -->
-
- <dependency>
- <groupId>org.hectorclient</groupId>
- <artifactId>hector-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-expression</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-beans</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-aop</artifactId>
- </dependency>
-
- <dependency>
- <groupId>aopalliance</groupId>
- <artifactId>aopalliance</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context-support</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-web</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.yaml</groupId>
- <artifactId>snakeyaml</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.jsoup</groupId>
- <artifactId>jsoup</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.perf4j</groupId>
- <artifactId>perf4j</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.aspectj</groupId>
- <artifactId>aspectjweaver</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.aspectj</groupId>
- <artifactId>aspectjrt</artifactId>
- </dependency>
-
- <dependency>
- <groupId>cglib</groupId>
- <artifactId>cglib-nodep</artifactId>
- </dependency>
-
- <dependency>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.jboss.netty</groupId>
- <artifactId>netty</artifactId>
- </dependency>
-
- <!-- Test and Logging Dependencies -->
-
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>usergrid-config</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
-
- <dependency>
- <groupId>org.hectorclient</groupId>
- <artifactId>hector-test</artifactId>
- <scope>test</scope>
- </dependency>
-
-
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.jvnet.mock-javamail</groupId>
- <artifactId>mock-javamail</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>usergrid-test-utils</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-test</artifactId>
- <scope>test</scope>
- </dependency>
-
-
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>common</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
-
- <!-- Core Persistence deps -->
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>collection</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- <type>jar</type>
- <exclusions>
- <exclusion>
- <artifactId>antlr</artifactId>
- <groupId>org.antlr</groupId>
- </exclusion>
- <exclusion>
- <artifactId>antlr</artifactId>
- <groupId>antlr</groupId>
- </exclusion>
- </exclusions>
- </dependency>
-
-
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>queryindex</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- <type>jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>graph</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- <type>jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>map</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- <type>jar</type>
- </dependency>
-
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>cache</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>queue</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- <type>jar</type>
- </dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- <version>${metrics.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.codahale.metrics</groupId>
- <artifactId>metrics-graphite</artifactId>
- <version>${metrics.version}</version>
- </dependency>
-
- </dependencies>
-
- <!--
- Do not need jacoco profile here because we do not override the surefire plugin in this POM
- -->
++
+ <testResources>
+ <testResource>
+ <directory>src/test/resources</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>**/*.yaml</include>
+ <include>**/*.properties</include>
+ <include>**/*.xml</include>
+ </includes>
+ </testResource>
+ <testResource>
+ <directory>src/test/resources</directory>
+ <filtering>true</filtering>
+ <includes>
+ <include>largeentity.json</include>
+ </includes>
+ </testResource>
+ </testResources>
+
+ <plugins>
+
+ <!--
+ Do not need to configure surefire plugin here, parent POM configuration is sufficient.
+ -->
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>test-jar-execution</id>
+ <phase>package</phase>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <configuration>
+ <includes>
+ <include>**/org/apache/usergrid/**</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <dependencies>
+
+
+ <!-- dependent on wiring guice and spring -->
+ <dependency>
+ <groupId>com.google.inject.extensions</groupId>
+ <artifactId>guice-spring</artifactId>
+ <version>4.0-beta5</version>
+ </dependency>
+
+ <!-- Apache Dependencies -->
+
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-all</artifactId>
+ <!-- Exclude the old and problematic Snappy -->
+ <exclusions>
+ <exclusion>
+ <artifactId>snappy-java</artifactId>
+ <groupId>org.xerial.snappy</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>antlr</artifactId>
+ <groupId>org.antlr</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>netty</artifactId>
+ <groupId>io.netty</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.cassandra</groupId>
+ <artifactId>cassandra-thrift</artifactId>
+ </dependency>
+
+ <!-- Include the slighly newer and less problematic snappy -->
+ <dependency>
+ <artifactId>snappy-java</artifactId>
+ <groupId>org.xerial.snappy</groupId>
+ <version>1.0.5</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-collections</groupId>
+ <artifactId>commons-collections</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ </dependency>
+
+ <!-- SUN, Javax Package, and Other Commercial Dependencies -->
+
+ <dependency>
+ <groupId>com.beust</groupId>
+ <artifactId>jcommander</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.sun.mail</groupId>
+ <artifactId>javax.mail</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.persistence</groupId>
+ <artifactId>persistence-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.uuid</groupId>
+ <artifactId>java-uuid-generator</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.hazelcast</groupId>
+ <artifactId>hazelcast-all</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.netflix.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.netflix.astyanax</groupId>
+ <artifactId>astyanax-recipes</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+
+ <!-- Codehaus, Spring and Other Org Dependencies -->
+
+ <dependency>
+ <groupId>org.hectorclient</groupId>
+ <artifactId>hector-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-expression</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-aop</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>aopalliance</groupId>
+ <artifactId>aopalliance</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context-support</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-web</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.yaml</groupId>
+ <artifactId>snakeyaml</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.jsoup</groupId>
+ <artifactId>jsoup</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.perf4j</groupId>
+ <artifactId>perf4j</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjweaver</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.aspectj</groupId>
+ <artifactId>aspectjrt</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>cglib</groupId>
+ <artifactId>cglib-nodep</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </dependency>
+
+ <!--
+ <dependency>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </dependency>
+ -->
+
+ <!-- Test and Logging Dependencies -->
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-config</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.hectorclient</groupId>
+ <artifactId>hector-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.jvnet.mock-javamail</groupId>
+ <artifactId>mock-javamail</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>usergrid-test-utils</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-test</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+
+ <!-- Core Persistence deps -->
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>collection</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ <exclusions>
+ <exclusion>
+ <artifactId>antlr</artifactId>
+ <groupId>org.antlr</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>antlr</artifactId>
+ <groupId>antlr</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>queryindex</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>graph</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>map</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>cache</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>actorsystem</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>queue</artifactId>
+ <version>${project.version}</version>
+ <type>jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.codahale.metrics</groupId>
+ <artifactId>metrics-graphite</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+
+ </dependencies>
+
+ <!--
+ Do not need jacoco profile here because we do not override the surefire plugin in this POM
+ -->
</project>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/849df674/stack/corepersistence/collection/pom.xml
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/pom.xml
index 2728fb4,88b96b1..20f8612
--- a/stack/corepersistence/collection/pom.xml
+++ b/stack/corepersistence/collection/pom.xml
@@@ -3,46 -19,88 +19,88 @@@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>persistence</artifactId>
- <groupId>org.apache.usergrid</groupId>
- <version>2.2.0-SNAPSHOT</version>
- </parent>
+ <parent>
+ <artifactId>persistence</artifactId>
+ <groupId>org.apache.usergrid</groupId>
- <version>2.1.1-SNAPSHOT</version>
++ <version>2.2.0-SNAPSHOT</version>
+ </parent>
+
+ <modelVersion>4.0.0</modelVersion>
+ <description>The module for handling all scope I/O</description>
+
+ <artifactId>collection</artifactId>
+ <name>Usergrid Collection</name>
+
+ <dependencies>
+
+ <!-- Google Guice Integration Test Injectors -->
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>actorsystem</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-actor_2.11</artifactId>
+ <version>2.4.0</version>
+ </dependency>
- <modelVersion>4.0.0</modelVersion>
- <description>The module for handling all scope I/O</description>
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-remote_2.11</artifactId>
+ <version>2.4.0</version>
+ </dependency>
- <artifactId>collection</artifactId>
- <name>Usergrid Collection</name>
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-cluster_2.11</artifactId>
+ <version>2.4.0</version>
+ </dependency>
- <dependencies>
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-cluster-tools_2.11</artifactId>
+ <version>2.4.0</version>
+ </dependency>
- <!-- Google Guice Integration Test Injectors -->
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-cluster-metrics_2.11</artifactId>
+ <version>2.4.0</version>
+ </dependency>
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>common</artifactId>
- <version>${project.version}</version>
- </dependency>
+ <dependency>
+ <groupId>com.typesafe.akka</groupId>
+ <artifactId>akka-slf4j_2.11</artifactId>
+ <version>2.4.0</version>
+ </dependency>
- <!-- tests -->
+ <!-- tests -->
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>common</artifactId>
- <version>${project.version}</version>
- <classifier>tests</classifier>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.apache.usergrid</groupId>
+ <artifactId>common</artifactId>
+ <version>${project.version}</version>
+ <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${mockito.version}</version>
- <scope>test</scope>
- </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
- </dependencies>
+ </dependencies>
<!--
<profiles>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/849df674/stack/corepersistence/common/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/849df674/stack/corepersistence/model/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/849df674/stack/corepersistence/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/849df674/stack/corepersistence/queryindex/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/849df674/stack/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/849df674/stack/services/pom.xml
----------------------------------------------------------------------
[25/50] usergrid git commit: Fix ports in REST tests too.
Posted by mr...@apache.org.
Fix ports in REST tests too.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/557ea914
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/557ea914
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/557ea914
Branch: refs/heads/master
Commit: 557ea91433dc3ffd8b78b89888fa0349226a860a
Parents: 5ddb1cd
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jul 1 16:41:47 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jul 1 16:41:47 2016 -0400
----------------------------------------------------------------------
.../src/test/resources/usergrid.properties | 3 ++-
.../test/resources/usergrid-custom-test.properties | 16 +++++++---------
2 files changed, 9 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/557ea914/stack/corepersistence/collection/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid.properties b/stack/corepersistence/collection/src/test/resources/usergrid.properties
index 8c999e2..ebc2ad4 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid.properties
@@ -20,8 +20,9 @@
# This property is required to be set and cannot be defaulted anywhere
usergrid.cluster_name=usergrid
-usergrid.cluster.hostname=localhost
+
usergrid.cluster.enabled=true
+usergrid.cluster.hostname=localhost
usergrid.cluster.region.local=us-east
usergrid.cluster.region.list=us-east
usergrid.cluster.seeds=us-east\:localhost
http://git-wip-us.apache.org/repos/asf/usergrid/blob/557ea914/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 31c6604..f20f1e5 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -56,14 +56,12 @@ usergrid.cluster_name=usergrid
usergrid.graph.shard.size=100
usergrid.graph.shard.repair.chance=1.0
-collection.akka.enabled=true
-collection.akka.hostname=localhost
-collection.akka.region=us-east
-usergrid.queue.regionList=us-east
-collection.akka.authoritative.region=us-east
-collection.akka.region.seeds=us-east\:localhost\:2551
+usergrid.cluster.enabled=true
+usergrid.cluster.hostname=localhost
+usergrid.cluster.region.local=us-east
+usergrid.cluster.region.list=us-east
+usergrid.cluster.seeds=us-east\:localhost
-collection.akka.uniquevalue.actors=300
-collection.akka.uniquevalue.cache.ttl=10
-collection.akka.uniquevalue.reservation.ttl=10
+collection.uniquevalues.actors=300
+collection.uniquevalues.authoritative.region=us-east
[48/50] usergrid git commit: Ensure the EntityIndexFactory is a
singleton so the cache is actually useful.
Posted by mr...@apache.org.
Ensure the EntityIndexFactory is a singleton so the cache is actually useful.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8faad5bd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8faad5bd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8faad5bd
Branch: refs/heads/master
Commit: 8faad5bdcbee9012f936cea6e8ef673cc5d07d54
Parents: 4f8183f
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Jul 12 15:48:32 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Jul 12 15:48:32 2016 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8faad5bd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index f4e89ad..175b08a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.concurrent.ExecutionException;
+import com.google.inject.Singleton;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.*;
@@ -32,6 +33,7 @@ import com.google.inject.Inject;
/**
* Get index from factory, adds caching
*/
+@Singleton
public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final IndexFig config;
[26/50] usergrid git commit: Clean up unique value scanner and let it
get consistently level from the typically read CL property.
Posted by mr...@apache.org.
Clean up unique value scanner and let it get consistently level from the typically read CL property.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ada914ac
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ada914ac
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ada914ac
Branch: refs/heads/master
Commit: ada914ac32ade7396f1b4e0f33d147ccbf328137
Parents: 3bba425
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jul 1 14:55:08 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jul 1 14:55:08 2016 -0700
----------------------------------------------------------------------
.../usergrid/tools/UniqueValueScanner.java | 105 +++++++++++--------
1 file changed, 60 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ada914ac/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
index 182e692..68f366b 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
@@ -18,9 +18,11 @@ package org.apache.usergrid.tools;
import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ConsistencyLevel;
import com.netflix.astyanax.util.RangeBuilder;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
@@ -131,6 +133,7 @@ public class UniqueValueScanner extends ToolBase {
}
logger.info("Staring Tool: UniqueValueScanner");
+ logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"));
keyspace = injector.getInstance(com.netflix.astyanax.Keyspace.class);
@@ -142,6 +145,8 @@ public class UniqueValueScanner extends ToolBase {
String entityType = line.getOptionValue(ENTITY_TYPE_ARG);
String entityName = line.getOptionValue(ENTITY_NAME_ARG);
+ AtomicInteger count = new AtomicInteger(0);
+
if (entityName != null && !entityName.isEmpty()) {
if(appToFilter == null){
@@ -157,11 +162,11 @@ public class UniqueValueScanner extends ToolBase {
logger.info("Running entity unique load only");
- //do stuff
+ //do stuff w/o read repair
UniqueValueSet uniqueValueSet = uniqueValueSerializationStrategy.load(
new ApplicationScopeImpl( new SimpleId(appToFilter, "application" ) ),
- entityType,
- Collections.singletonList(new StringField( fieldType, entityName) ));
+ ConsistencyLevel.valueOf(System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")), entityType,
+ Collections.singletonList(new StringField( fieldType, entityName) ), false);
StringBuilder stringBuilder = new StringBuilder();
@@ -194,89 +199,99 @@ public class UniqueValueScanner extends ToolBase {
try {
rows = keyspace.prepareQuery(CF_UNIQUE_VALUES)
+ .setConsistencyLevel(ConsistencyLevel.valueOf(System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")))
.getAllRows()
- .withColumnRange(new RangeBuilder().build())
+ .withColumnRange(new RangeBuilder().setLimit(1000).build())
.execute().getResult().iterator();
} catch (ConnectionException e) {
+ logger.error("Error connecting to cassandra", e);
}
UUID finalAppToFilter = appToFilter;
- rows.forEachRemaining(row -> {
- final String fieldName = row.getKey().getKey().getField().getName();
- final String fieldValue = row.getKey().getKey().getField().getValue().toString();
- final String scopeType = row.getKey().getScope().getType();
- final UUID scopeUUID = row.getKey().getScope().getUuid();
+ if( rows != null) {
+ rows.forEachRemaining(row -> {
+ count.incrementAndGet();
- if (!fieldName.equalsIgnoreCase(fieldType) ||
- (finalAppToFilter != null && !finalAppToFilter.equals(scopeUUID))
- ) {
- // do nothing
+ if(count.get() % 1000 == 0 ){
+ logger.info("Scanned {} rows in {}", count.get(), CF_UNIQUE_VALUES.getName());
+ }
- } else {
+ final String fieldName = row.getKey().getKey().getField().getName();
+ final String fieldValue = row.getKey().getKey().getField().getValue().toString();
+ final String scopeType = row.getKey().getScope().getType();
+ final UUID scopeUUID = row.getKey().getScope().getUuid();
- if (em == null && finalAppToFilter.equals(scopeUUID)) {
- em = emf.getEntityManager(scopeUUID);
- }
- // if we have more than 1 column, let's check for a duplicate
- if(row.getColumns().size() > 1) {
+ if (!fieldName.equalsIgnoreCase(fieldType) ||
+ (finalAppToFilter != null && !finalAppToFilter.equals(scopeUUID))
+ ) {
+ // do nothing
- final List<EntityVersion> values = new ArrayList<>(row.getColumns().size());
+ } else {
- Iterator<Column<EntityVersion>> columns = row.getColumns().iterator();
- columns.forEachRemaining(column -> {
+ // if we have more than 1 column, let's check for a duplicate
+ if (row.getColumns() != null && row.getColumns().size() > 1) {
+ final List<EntityVersion> values = new ArrayList<>(row.getColumns().size());
- final EntityVersion entityVersion = column.getName();
+ Iterator<Column<EntityVersion>> columns = row.getColumns().iterator();
+ columns.forEachRemaining(column -> {
+ final EntityVersion entityVersion = column.getName();
- logger.trace(
- scopeType + ": " + scopeUUID + ", " +
- fieldName + ": " + fieldValue + ", " +
- "entity type: " + entityVersion.getEntityId().getType() + ", " +
- "entity uuid: " + entityVersion.getEntityId().getUuid()
- );
+ logger.trace(
+ scopeType + ": " + scopeUUID + ", " +
+ fieldName + ": " + fieldValue + ", " +
+ "entity type: " + entityVersion.getEntityId().getType() + ", " +
+ "entity uuid: " + entityVersion.getEntityId().getUuid()
+ );
- if (entityType != null &&
- entityVersion.getEntityId().getType().equalsIgnoreCase(entityType)
- ) {
- // add the first value into the list
- if(values.size() == 0 ) {
+ if (entityType != null &&
+ entityVersion.getEntityId().getType().equalsIgnoreCase(entityType)
+ ) {
- values.add(entityVersion);
+ // add the first value into the list
+ if (values.size() == 0) {
+ values.add(entityVersion);
- }else{
- if( !values.get(0).getEntityId().getUuid().equals(entityVersion.getEntityId().getUuid())){
+ } else {
- values.add(entityVersion);
+ if (!values.get(0).getEntityId().getUuid().equals(entityVersion.getEntityId().getUuid())) {
+
+ values.add(entityVersion);
- logger.error("Duplicate found for field [{}={}]. Entry 1: [{}], Entry 2: [{}]",
- fieldName, fieldValue, values.get(0).getEntityId(), entityVersion.getEntityId());
+ logger.error("Duplicate found for field [{}={}]. Entry 1: [{}], Entry 2: [{}]",
+ fieldName, fieldValue, values.get(0).getEntityId(), entityVersion.getEntityId());
+
+ }
}
+
}
+ });
+ }
+ }
- }
- });
- }
- }
+ });
+ }else{
+ logger.warn("No rows returned from table: {}", CF_UNIQUE_VALUES.getName());
- });
+ }
}
}
[21/50] usergrid git commit: Simplify and rename Actorsystem
configuration properties to be more generic,
e.g. starting with "usergrid.cluster" instead of "collection.akka"
Posted by mr...@apache.org.
Simplify and rename Actorsystem configuration properties to be more generic, e.g. starting with "usergrid.cluster" instead of "collection.akka"
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f0c9fd4b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f0c9fd4b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f0c9fd4b
Branch: refs/heads/master
Commit: f0c9fd4bd91a271ee1e9a93a6fa70bf69159f7db
Parents: 2d5ad05
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jul 1 11:09:37 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jul 1 11:09:37 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 56 ++---
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../persistence/actorsystem/ActorSystemFig.java | 60 ++---
.../actorsystem/ActorSystemManagerImpl.java | 53 ++---
.../mvcc/stage/write/WriteCommit.java | 10 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 10 +-
.../uniquevalues/UniqueValuesFig.java | 28 +--
.../mvcc/stage/delete/MarkCommitTest.java | 2 +-
.../mvcc/stage/write/WriteCommitTest.java | 8 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +-
.../org/apache/usergrid/rest/UniqueCatsIT.java | 237 +++++++++++++++++++
11 files changed, 344 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 29b8d36..fe70569 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -410,60 +410,44 @@ usergrid.queue.lock.timeout=5
#usergrid.queue.publish.queuesize=850000
-######################### Akka Actor System Configiuration ###################
+######################### Usergrid Cluster Configuration ###################
#
-# Usergrid includes Akka, an Actor-based system that allows for the
-# distribution of work across multiple Usergrid instances and multiple regions.
-#
-# All properties are required. If Akka is enabled then all properties in this
-# section MUST be specified.
-#
-# For more information: https://issues.apache.org/jira/browse/USERGRID-1268
+# Usergrid includes a multi-region clustering system.
+# To user it you must specify your region, the list of regions and seeds for each region.
#
-# Currently, Akka is disable and not required for Usergrid
-collection.akka.enabled=false
+# This is an experimentation new feature, disabled by default
+usergrid.cluster.enabled=false
-# host name of this machine
-collection.akka.hostname=localhost
+# Comma-separated list of regions to be considered
+usergrid.cluster.region.list=default
-# The region of this Usergrid installation
-# Region MUST be in the region list specified in the 'usergrid.queue.regionList' property
-collection.akka.region=
+# The regions of this local instance of Usergrid
+usergrid.cluster.region.local=default
-# Comma-separated lists of Akka seeds each with format {region}:{hostname}:{port}.
-# All regions MUST be listed in the 'usergrid.queue.regionList'
-collection.akka.region.seeds=
-
-# The default authoritative region for when is not specified elsewhere
-# Region MUST be in the region list specified in the 'usergrid.queue.regionList' property
-collection.akka.authoritative.region=
+# Comma-separated lists of cluster seeds each with format {region}:{hostname}
+usergrid.cluster.seeds=default:localhost
-# Default number of Akka actors to start per instance / router producer
-collection.akka.instances-per-node=300
+# Port used for cluster communications.
+usergrid.cluster.port=2551
######################### Usergrid Unique Values Validation ##################
#
-# Usergrid includes a distributed unique values validation that ensure that
-# unique values rename unique across a distributed and multi-region system.
-# This system is based on the Akka actor system and requires some additional
-# configuration.
-#
-# The system uses consistent hashing to ensure that one single-threaded actor
-# ever accesses a unique value record at one time.
-#
-# For more information: https://issues.apache.org/jira/browse/USERGRID-1268
+# These only apply if the above Usergrid cluster system is enabled.
#
# The number of unique value actors to start on each Usergrid instance.
-collection.akka.uniquevalue.actors=300
+collection.uniquevalues.actors=300
# TTL of unique value reservation in in-memory cache
-collection.akka.uniquevalue.cache.ttl=10
+collection.uniquevalues.cache.ttl=10
# TTL of a unique value reservation when written to Cassandra
-collection.akka.uniquevalue.reservation.ttl=10
+collection.uniquevalues.reservation.ttl=10
+
+# The default authoritative region for when is not specified elsewhere
+collection.uniquevalues.authoritative.region=default
############################## Usergrid Scheduler ###########################
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index eca5927..e70a6fd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -142,7 +142,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
logger.info("EntityManagerFactoring starting...");
- if ( actorSystemFig.getAkkaEnabled() ) {
+ if ( actorSystemFig.getEnabled() ) {
try {
logger.info("Akka cluster starting...");
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemFig.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemFig.java
index ec010d0..5d7b6aa 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemFig.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemFig.java
@@ -30,64 +30,54 @@ import java.io.Serializable;
@FigSingleton
public interface ActorSystemFig extends GuicyFig, Serializable {
- String AKKA_ENABLED = "collection.akka.enabled";
+ String CLUSTER_ENABLED = "usergrid.cluster.enabled";
- String AKKA_HOSTNAME = "collection.akka.hostname";
+ String CLUSTER_REGIONS_LIST = "usergrid.cluster.region.list";
- String AKKA_REGION = "collection.akka.region";
+ String CLUSTER_REGIONS_LOCAL = "usergrid.cluster.region.local";
- String AKKA_REGION_LIST = "usergrid.queue.regionList"; // same region list used by queues
+ String CLUSTER_SEEDS = "usergrid.cluster.seeds";
- String AKKA_REGION_SEEDS = "collection.akka.region.seeds";
-
- String AKKA_AUTHORITATIVE_REGION = "collection.akka.authoritative.region";
-
- String AKKA_INSTANCES_PER_NODE = "collection.akka.instances-per-node";
+ String CLUSTER_PORT = "usergrid.cluster.port";
/**
- * Use Akka or nah
+ * Use Cluster or nah
*/
- @Key(AKKA_ENABLED)
+ @Key(CLUSTER_ENABLED)
@Default("true")
- boolean getAkkaEnabled();
-
- /**
- * Hostname to be used in Akka configuration.
- */
- @Key(AKKA_HOSTNAME)
- String getHostname();
+ boolean getEnabled();
/**
* Local region to be used in Akka configuration.
*/
- @Key(AKKA_REGION)
- String getRegion();
+ @Key(CLUSTER_REGIONS_LOCAL)
+ @Default("default")
+ String getRegionLocal();
/**
* Comma separated list of regions known to cluster.
*/
- @Key(AKKA_REGION_LIST)
- String getRegionList();
+ @Key(CLUSTER_REGIONS_LIST)
+ @Default("default")
+ String getRegionsList();
/**
- * Comma-separated lists of seeds each with format {region}:{hostname}:{port}.
- * Regions MUST be listed in the 'usergrid.queue.regionList'
+ * Comma-separated lists of seeds each with format {region}:{hostname}
*/
- @Key(AKKA_REGION_SEEDS)
- String getRegionSeeds();
+ @Key(CLUSTER_SEEDS)
+ @Default("default:localhost")
+ String getSeeds();
/**
- * If no region specified for type, use the authoritative region
+ * Port for cluster comms.
*/
- @Key(AKKA_AUTHORITATIVE_REGION)
- String getAkkaAuthoritativeRegion();
+ @Key(CLUSTER_PORT)
+ @Default("2551")
+ String getPort();
- /**
- * Number of actor instances to create on each node for each router.
- */
- @Key(AKKA_INSTANCES_PER_NODE)
- @Default("300")
- int getInstancesPerNode();
+ @Key("usergrid.cluster.hostname")
+ @Default("")
+ String getHostname();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index 05f837d..a79f447 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -39,6 +39,8 @@ import org.slf4j.LoggerFactory;
import scala.concurrent.Await;
import scala.concurrent.Future;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.TimeUnit;
@@ -77,8 +79,17 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
@Override
public void start() {
- this.hostname = actorSystemFig.getHostname();
- this.currentRegion = actorSystemFig.getRegion();
+ if ( !StringUtils.isEmpty( actorSystemFig.getHostname()) ) {
+ this.hostname = actorSystemFig.getHostname();
+ } else {
+ try {
+ this.hostname = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException e) {
+ logger.error("Cannot get hostname, defaulting to 'localhost': " + e.getMessage());
+ }
+ }
+
+ this.currentRegion = actorSystemFig.getRegionLocal();
this.port = null;
initAkka();
@@ -155,32 +166,22 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
// Create one actor system with request actor for each region
- if ( StringUtils.isEmpty( hostname )) {
- throw new RuntimeException( "No value specified for " + ActorSystemFig.AKKA_HOSTNAME );
- }
-
if ( StringUtils.isEmpty( currentRegion )) {
- throw new RuntimeException( "No value specified for " + ActorSystemFig.AKKA_REGION );
+ throw new RuntimeException( "No value specified for " + ActorSystemFig.CLUSTER_REGIONS_LOCAL );
}
- if ( StringUtils.isEmpty( actorSystemFig.getRegionList() )) {
- throw new RuntimeException( "No value specified for " + ActorSystemFig.AKKA_REGION_LIST );
+ if ( StringUtils.isEmpty( actorSystemFig.getRegionsList() )) {
+ throw new RuntimeException( "No value specified for " + ActorSystemFig.CLUSTER_REGIONS_LIST );
}
- if ( StringUtils.isEmpty( actorSystemFig.getRegionSeeds() )) {
- throw new RuntimeException( "No value specified for " + ActorSystemFig.AKKA_REGION_SEEDS);
+ if ( StringUtils.isEmpty( actorSystemFig.getSeeds() )) {
+ throw new RuntimeException( "No value specified for " + ActorSystemFig.CLUSTER_SEEDS );
}
- if ( StringUtils.isEmpty( actorSystemFig.getAkkaAuthoritativeRegion() )) {
- logger.warn("No value for {} specified, will use current region as authoriative region",
- ActorSystemFig.AKKA_AUTHORITATIVE_REGION);
- //throw new RuntimeException( "No value specified for " + ActorSystemFig.AKKA_AUTHORITATIVE_REGION);
- }
-
- List regionList = Arrays.asList( actorSystemFig.getRegionList().toLowerCase().split(",") );
+ List regionList = Arrays.asList( actorSystemFig.getRegionsList().toLowerCase().split(",") );
logger.info("Initializing Akka for hostname {} region {} regionList {} seeds {}",
- hostname, currentRegion, regionList, actorSystemFig.getRegionSeeds() );
+ hostname, currentRegion, regionList, actorSystemFig.getSeeds() );
Config config = readClusterSystemConfig();
@@ -205,7 +206,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
seedsByRegion = ArrayListMultimap.create();
- String[] regionSeeds = actorSystemFig.getRegionSeeds().split( "," );
+ String[] regionSeeds = actorSystemFig.getSeeds().split( "," );
logger.info( "Found region {} seeds {}", regionSeeds.length, regionSeeds );
@@ -226,7 +227,8 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
String[] parts = regionSeed.split( ":" );
String region = parts[0];
String hostname = parts[1];
- String regionPortString = parts[2];
+
+ String regionPortString = parts.length > 2 ? parts[2] : actorSystemFig.getPort();
// all seeds in same region must use same port
// we assume 0th seed has the right port
@@ -269,7 +271,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
try {
- int numInstancesPerNode = actorSystemFig.getInstancesPerNode();
+ int numInstancesPerNode = 300; // expect this to be overridden by RouterProducers
String region = currentRegion;
@@ -277,11 +279,8 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
int lastColon = seeds.get(0).lastIndexOf(":") + 1;
final Integer regionPort = Integer.parseInt( seeds.get(0).substring( lastColon ));
- logger.info( "Akka Config for region {} is:\n" +
- " Hostname {}\n" +
- " Seeds {}\n" +
- " Authoritative Region {}\n",
- region, hostname, seeds, actorSystemFig.getAkkaAuthoritativeRegion() );
+ logger.info( "Akka Config for region {} is:\n" + " Hostname {}\n" + " Seeds {}\n",
+ region, hostname, seeds );
Map<String, Object> configMap = new HashMap<String, Object>() {{
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
index 65d1734..5b98ca5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
@@ -25,6 +25,7 @@ import java.util.UUID;
import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueException;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesFig;
import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,6 +68,7 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Collect
private static final Logger logger = LoggerFactory.getLogger( WriteCommit.class );
ActorSystemFig actorSystemFig;
+ UniqueValuesFig uniqueValuesFig;
UniqueValuesService akkaUvService;
@Inject
@@ -82,6 +84,7 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Collect
final MvccEntitySerializationStrategy entryStrat,
final UniqueValueSerializationStrategy uniqueValueStrat,
final ActorSystemFig actorSystemFig,
+ final UniqueValuesFig uniqueValuesFig,
final UniqueValuesService akkaUvService ) {
Preconditions.checkNotNull( logStrat, "MvccLogEntrySerializationStrategy is required" );
@@ -92,6 +95,7 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Collect
this.entityStrat = entryStrat;
this.uniqueValueStrat = uniqueValueStrat;
this.actorSystemFig = actorSystemFig;
+ this.uniqueValuesFig = uniqueValuesFig;
this.akkaUvService = akkaUvService;
}
@@ -130,13 +134,13 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Collect
logMutation.mergeShallow( entityMutation );
// akkaFig may be null when this is called from JUnit tests
- if ( actorSystemFig != null && actorSystemFig.getAkkaEnabled() ) {
+ if ( actorSystemFig != null && actorSystemFig.getEnabled() ) {
String region = ioEvent.getRegion();
if ( region == null ) {
- region = actorSystemFig.getAkkaAuthoritativeRegion();
+ region = uniqueValuesFig.getAuthoritativeRegion();
}
if ( region == null ) {
- region = actorSystemFig.getRegion();
+ region = actorSystemFig.getRegionLocal();
}
confirmUniqueFieldsAkka( mvccEntity, version, applicationScope, region );
} else {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index f159096..985137b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -40,6 +40,7 @@ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSeria
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueException;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesFig;
import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -63,6 +64,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
private static final Logger logger = LoggerFactory.getLogger( WriteUniqueVerify.class );
ActorSystemFig actorSystemFig;
+ UniqueValuesFig uniqueValuesFig;
UniqueValuesService akkaUvService;
private final UniqueValueSerializationStrategy uniqueValueStrat;
@@ -83,11 +85,13 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
final Keyspace keyspace,
final CassandraConfig cassandraFig,
final ActorSystemFig actorSystemFig,
+ final UniqueValuesFig uniqueValuesFig,
final UniqueValuesService akkaUvService ) {
this.keyspace = keyspace;
this.cassandraFig = cassandraFig;
this.actorSystemFig = actorSystemFig;
+ this.uniqueValuesFig = uniqueValuesFig;
this.akkaUvService = akkaUvService;
Preconditions.checkNotNull( uniqueValueSerializiationStrategy, "uniqueValueSerializationStrategy is required" );
@@ -102,7 +106,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
@Override
public void call( final CollectionIoEvent<MvccEntity> ioevent ) {
- if ( actorSystemFig != null && actorSystemFig.getAkkaEnabled() ) {
+ if ( actorSystemFig != null && actorSystemFig.getEnabled() ) {
verifyUniqueFieldsAkka( ioevent );
} else {
verifyUniqueFields( ioevent );
@@ -121,10 +125,10 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
String region = ioevent.getRegion();
if ( region == null ) {
- region = actorSystemFig.getAkkaAuthoritativeRegion();
+ region = uniqueValuesFig.getAuthoritativeRegion();
}
if ( region == null ) {
- region = actorSystemFig.getRegion();
+ region = actorSystemFig.getRegionLocal();
}
try {
akkaUvService.reserveUniqueValues( applicationScope, entity, mvccEntity.getVersion(), region );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesFig.java
index c99824f..edd0cbe 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesFig.java
@@ -28,40 +28,40 @@ import java.io.Serializable;
@FigSingleton
public interface UniqueValuesFig extends GuicyFig, Serializable {
- String AKKA_UNIQUEVALUE_ACTORS = "collection.akka.uniquevalue.actors";
+ String UNIQUEVALUE_ACTORS = "collection.uniquevalues.actors";
- String AKKA_UNIQUEVALUE_CACHE_TTL = "collection.akka.uniquevalue.cache.ttl";
+ String UNIQUEVALUE_CACHE_TTL = "collection.uniquevalues.cache.ttl";
- String AKKA_UNIQUEVALUE_RESERVATION_TTL= "collection.akka.uniquevalue.reservation.ttl";
+ String UNIQUEVALUE_RESERVATION_TTL= "collection.uniquevalues.reservation.ttl";
- String AKKA_UNIQUEVALUE_INSTANCES_PER_NODE = "collection.akka.uniquevalue.instances-per-node";
+ String UNIQUEVALUE_AUTHORITATIVE_REGION = "collection.uniquevalues.authoritative.region";
/**
- * Number of UniqueValueActors to be started on each node
- */
- @Key(AKKA_UNIQUEVALUE_ACTORS)
- @Default("300")
- int getUniqueValueActors();
-
- /**
* Unique Value cache TTL in seconds.
*/
- @Key(AKKA_UNIQUEVALUE_CACHE_TTL)
+ @Key(UNIQUEVALUE_CACHE_TTL)
@Default("10")
int getUniqueValueCacheTtl();
/**
* Unique Value Reservation TTL in seconds.
*/
- @Key(AKKA_UNIQUEVALUE_RESERVATION_TTL)
+ @Key(UNIQUEVALUE_RESERVATION_TTL)
@Default("10")
int getUniqueValueReservationTtl();
/**
* Number of actor instances to create on each.
*/
- @Key(AKKA_UNIQUEVALUE_INSTANCES_PER_NODE)
+ @Key(UNIQUEVALUE_ACTORS)
@Default("300")
int getUniqueValueInstancesPerNode();
+
+ /**
+ * Primary authoritative region (used if none other specified).
+ */
+ @Key(UNIQUEVALUE_AUTHORITATIVE_REGION)
+ @Default("default")
+ String getAuthoritativeRegion();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
index e7cee21..a0ee6be 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
@@ -71,7 +71,7 @@ public class MarkCommitTest extends AbstractMvccEntityStageTest {
//run the stage
- WriteCommit newStage = new WriteCommit( logStrategy, mvccEntityStrategy, uniqueValueStrategy, null, null );
+ WriteCommit newStage = new WriteCommit( logStrategy, mvccEntityStrategy, uniqueValueStrategy, null, null, null);
//verify the observable is correct
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java
index 8665ee9..dcc473c 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java
@@ -84,10 +84,12 @@ public class WriteCommitTest extends AbstractMvccEntityStageTest {
//run the stage
- WriteCommit newStage = new WriteCommit( logStrategy, mvccEntityStrategy, uniqueValueStrategy, null, null );
+ WriteCommit newStage =
+ new WriteCommit( logStrategy, mvccEntityStrategy, uniqueValueStrategy, null, null, null );
- Entity result = newStage.call( new CollectionIoEvent<MvccEntity>( context, mvccEntityInput ) ).getEvent().getEntity().get();
+ Entity result = newStage.call(
+ new CollectionIoEvent<MvccEntity>( context, mvccEntityInput ) ).getEvent().getEntity().get();
//verify the log entry is correct
@@ -131,7 +133,7 @@ public class WriteCommitTest extends AbstractMvccEntityStageTest {
when( mvccEntityStrategy.write( any( ApplicationScope.class ), any( MvccEntity.class ) ) )
.thenReturn( entityMutation );
- new WriteCommit( logStrategy, mvccEntityStrategy, uniqueValueStrategy, null, null ).call( event );
+ new WriteCommit( logStrategy, mvccEntityStrategy, uniqueValueStrategy, null, null, null ).call( event );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
index 635e262..46cfde1 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
@@ -96,7 +96,7 @@ public class WriteUniqueVerifyTest extends AbstractUniqueValueTest {
final MvccEntity mvccEntity = fromEntity( entity );
// run the stage
- WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace, cassandraConfig, null, null );
+ WriteUniqueVerify newStage = new WriteUniqueVerify( uvstrat, fig, keyspace, cassandraConfig, null, null, null );
newStage.call( new CollectionIoEvent<>( collectionScope, mvccEntity ) ) ;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f0c9fd4b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
new file mode 100644
index 0000000..0120660
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import org.apache.commons.lang.RandomStringUtils;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.ProcessingException;
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.Entity;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import java.net.ConnectException;
+import java.text.DecimalFormat;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static com.codahale.metrics.MetricRegistry.name;
+
+
+/**
+ * Tests that Catgrid will not allow creation of entities with duplicate names.
+ *
+ * Intended for use against a production-like cluster, not run during normal JUnit testing.
+ *
+ * Comment out the @Ignore annotation below and edit to add your target hosts.
+ */
+public class UniqueCatsIT {
+ private static final Logger logger = LoggerFactory.getLogger( UniqueCatsIT.class );
+
+ private static final AtomicInteger successCounter = new AtomicInteger( 0 );
+ private static final AtomicInteger errorCounter = new AtomicInteger( 0 );
+ private static final AtomicInteger dupCounter = new AtomicInteger( 0 );
+
+ @Test
+ //@Ignore("Intended for use against prod-like cluster")
+ public void testDuplicatePrevention() throws Exception {
+
+ int numThreads = 20;
+ int poolSize = 20;
+ int numCats = 100;
+
+ Multimap<String, String> catsCreated = Multimaps.synchronizedMultimap( HashMultimap.create() );
+ Multimap<String, Map<String, Object>> dupsRejected = Multimaps.synchronizedMultimap( HashMultimap.create() );
+
+ ExecutorService execService = Executors.newFixedThreadPool( poolSize );
+
+ Client client = ClientBuilder.newClient();
+
+ final MetricRegistry metrics = new MetricRegistry();
+ final Timer responses = metrics.timer(name(UniqueCatsIT.class, "responses"));
+ long startTime = System.currentTimeMillis();
+
+ final AtomicBoolean failed = new AtomicBoolean(false);
+
+ //String[] targetHosts = {"http://localhost:8080"};
+
+ String[] targetHosts = {
+ "https://ug21-west.e2e.apigee.net",
+ "https://ug21-east.e2e.apigee.net"
+ };
+
+ for (int i = 0; i < numCats; i++) {
+
+ if ( failed.get() ) { break; }
+
+ String randomizer = RandomStringUtils.randomAlphanumeric( 8 );
+
+ // multiple threads simultaneously trying to create a cat with the same propertyName
+ for (int j = 0; j < numThreads; j++) {
+
+ if ( failed.get() ) { break; }
+
+ final String name = "uv_test_cat_" + randomizer;
+ final String host = targetHosts[ j % targetHosts.length ];
+
+ execService.submit( () -> {
+
+ Map<String, Object> form = new HashMap<String, Object>() {{
+ put("name", name);
+ }};
+
+ Timer.Context time = responses.time();
+ try {
+ WebTarget target = client.target( host ).path(
+ //"/test-organization/test-app/cats" );
+ "/dmjohnson/sandbox/cats" );
+
+ //logger.info("Posting cat {} to host {}", catname, host);
+
+ Response response = target.request()
+ //.post( Entity.entity( form, MediaType.APPLICATION_FORM_URLENCODED ));
+ .post( Entity.entity( form, MediaType.APPLICATION_JSON));
+
+ org.apache.usergrid.rest.test.resource.model.ApiResponse apiResponse = null;
+ String responseAsString = "";
+ if ( response.getStatus() >= 400 ) {
+ responseAsString = response.readEntity( String.class );
+ } else {
+ apiResponse = response.readEntity(
+ org.apache.usergrid.rest.test.resource.model.ApiResponse.class );
+ }
+
+ if ( response.getStatus() == 200 || response.getStatus() == 201 ) {
+ catsCreated.put( name, apiResponse.getEntity().getUuid().toString() );
+ successCounter.incrementAndGet();
+
+ } else if ( response.getStatus() == 400
+ && responseAsString.contains("DuplicateUniquePropertyExistsException")) {
+ dupsRejected.put( name, form );
+ dupCounter.incrementAndGet();
+
+ } else {
+ logger.error("Cat creation failed status {} message {}",
+ response.getStatus(), responseAsString );
+ errorCounter.incrementAndGet();
+ }
+
+ } catch ( ProcessingException e ) {
+ errorCounter.incrementAndGet();
+ if ( e.getCause() instanceof ConnectException ) {
+ logger.error("Error connecting to " + host);
+ } else {
+ logger.error( "Error", e );
+ }
+
+ } catch ( Exception e ) {
+ errorCounter.incrementAndGet();
+ logger.error("Error", e);
+ }
+ time.stop();
+
+ } );
+ }
+ }
+ execService.shutdown();
+
+ try {
+ while (!execService.awaitTermination( 60, TimeUnit.SECONDS )) {
+ System.out.println( "Waiting..." );
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ long endTime = System.currentTimeMillis();
+
+ logger.info( "Total time {}s", (endTime - startTime) / 1000 );
+
+ DecimalFormat format = new DecimalFormat("##.###");
+
+ logger.info( "Timed {} requests:\n" +
+ "mean rate {}/s\n" +
+ "min {}s\n" +
+ "max {}s\n" +
+ "mean {}s",
+ responses.getCount(),
+ format.format( responses.getMeanRate() ),
+ format.format( (double)responses.getSnapshot().getMin() / 1000000000 ),
+ format.format( (double)responses.getSnapshot().getMax() / 1000000000 ),
+ format.format( responses.getSnapshot().getMean() / 1000000000 )
+ );
+
+ logger.info( "Error count {} ratio = {}",
+ errorCounter.get(), (float) errorCounter.get() / (float) responses.getCount() );
+
+ logger.info( "Success count = {}", successCounter.get() );
+
+ logger.info( "Rejected dup count = {}", dupCounter.get() );
+
+// for ( String catname : catsCreated.keys() ) {
+// System.out.println( catname );
+// Collection<Cat> cats = catsCreated.get( catname );
+// for ( Cat cat : cats ) {
+// System.out.println(" " + cat.getUuid() );
+// }
+// }
+
+// int count = 0;
+// for ( String catname : dupsRejected.keySet() ) {
+// System.out.println( catname );
+// Collection<Cat> cats = dupsRejected.get( catname );
+// for ( Cat cat : cats ) {
+// System.out.println(" " + (count++) + " rejected " + cat.getCatname() + ":" + cat.getUuid() );
+// }
+// }
+
+ int catCount = 0;
+ int catnamesWithDuplicates = 0;
+ for ( String name : catsCreated.keySet() ) {
+ //Collection<Map<String, String>> forms =
+ Collection<String> forms = catsCreated.get( name );
+ if ( forms.size() > 1 ) {
+ catnamesWithDuplicates++;
+ logger.info("Duplicate " + name);
+ }
+ catCount++;
+ }
+ Assert.assertEquals( 0, catnamesWithDuplicates );
+ Assert.assertEquals( 0, errorCounter.get() );
+ Assert.assertEquals( numCats, successCounter.get() );
+ Assert.assertEquals( numCats, catCount );
+
+
+ }
+
+}
[20/50] usergrid git commit: Make logging more readable.
Posted by mr...@apache.org.
Make logging more readable.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/61aa0377
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/61aa0377
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/61aa0377
Branch: refs/heads/master
Commit: 61aa03775eaca435d20d8a2e5af47d0dcb0f7641
Parents: 57231f0
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Jun 30 22:09:47 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Jun 30 22:09:47 2016 -0700
----------------------------------------------------------------------
.../impl/UniqueValueSerializationStrategyImpl.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61aa0377/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index f37e219..d7b1be9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -394,11 +394,14 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
}
// take the last candidate ( should be the latest version) and add to the result set
+
+ final UniqueValue returnValue = candidates.get(candidates.size() -1);
if(logger.isTraceEnabled()){
- logger.trace("Adding unique value [{}] to response set.",
- candidates.get(candidates.size() -1));
+ logger.trace("Adding unique value [{}={}] with entity id [{}] and entity version [{}] to response set",
+ returnValue.getField().getName(), returnValue.getField().getValue().toString(),
+ returnValue.getEntityId().getUuid(), returnValue.getEntityVersion());
}
- uniqueValueSet.addValue(candidates.get(candidates.size() -1));
+ uniqueValueSet.addValue(returnValue);
}
[14/50] usergrid git commit: Clean up em.removeFromCollection and fix
UniqueValueScanner to just log duplicates if scanning all of unique values.
Posted by mr...@apache.org.
Clean up em.removeFromCollection and fix UniqueValueScanner to just log duplicates if scanning all of unique values.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f4ae0cb8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f4ae0cb8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f4ae0cb8
Branch: refs/heads/master
Commit: f4ae0cb85b09547a2bb808a8a80fa8a36c485551
Parents: 426449e
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jun 29 10:55:14 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jun 29 10:55:14 2016 -0700
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 36 ++------------------
.../mvcc/stage/write/WriteUniqueVerify.java | 1 -
2 files changed, 2 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f4ae0cb8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index fbf0b14..d2ab702 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -525,41 +525,9 @@ public class CpRelationManager implements RelationManager {
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
- // remove edge from collection to item
- GraphManager gm = managerCache.getGraphManager( applicationScope );
-
-
-
- // mark the edge versions and take the first for later delete edge queue event ( load is descending )
- final Edge markedSourceEdge = gm.loadEdgeVersions(
- CpNamingUtils.createEdgeFromCollectionName( cpHeadEntity.getId(), collectionName, entityId ) )
- .flatMap(edge -> gm.markEdge(edge)).toBlocking().firstOrDefault(null);
-
-
- Edge markedReversedEdge = null;
- CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName );
- if (collection != null && collection.getLinkedCollection() != null) {
- // delete reverse edges
- final String pluralType = InflectionUtils.pluralize( cpHeadEntity.getId().getType() );
- markedReversedEdge = gm.loadEdgeVersions(
- CpNamingUtils.createEdgeFromCollectionName( entityId, pluralType, cpHeadEntity.getId() ) )
- .flatMap(reverseEdge -> gm.markEdge(reverseEdge)).toBlocking().firstOrDefault(null);
- }
-
-
- /**
- * Remove from the index. This will call gm.deleteEdge which also deletes the reverse edge(s) and de-indexes
- * older versions of the edge(s).
- *
- */
- if( markedSourceEdge != null ) {
- indexService.queueDeleteEdge(applicationScope, markedSourceEdge);
- }
- if( markedReversedEdge != null ){
- indexService.queueDeleteEdge(applicationScope, markedReversedEdge);
-
- }
+ // this will remove the edges from app->entity(collection)
+ removeItemFromCollection(collectionName, itemRef);
// special handling for roles collection of a group
if ( headEntity.getType().equals( Group.ENTITY_TYPE ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f4ae0cb8/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index d7c8ecd..adbe03d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -154,7 +154,6 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
if(logger.isTraceEnabled()){
logger.trace("Pre-write unique violations found, raising exception before executing first write");
}
- logger.error("Pre-write unique violations found, raising exception before executing first write");
throw new WriteUniqueVerifyException(mvccEntity, scope,
preWriteUniquenessViolations );
[41/50] usergrid git commit: Remove unnecessary releaseUniqueValues()
logic.
Posted by mr...@apache.org.
Remove unnecessary releaseUniqueValues() logic.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d5d89a82
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d5d89a82
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d5d89a82
Branch: refs/heads/master
Commit: d5d89a8203f8939fa2a436a718e3f5d4b3cce3de
Parents: 37972a2
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 13:53:45 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 13:53:45 2016 -0400
----------------------------------------------------------------------
.../mvcc/stage/delete/MarkCommit.java | 36 +++++++++++---------
.../uniquevalues/UniqueValuesService.java | 7 ++--
.../uniquevalues/UniqueValuesServiceImpl.java | 30 +++++++++-------
3 files changed, 40 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d5d89a82/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index 70c785e..57d92e1 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@ -126,23 +126,25 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
throw new RuntimeException( "Unable to mark entry as deleted" );
}
- // actorSystemFig may be null in testing
- if ( actorSystemFig != null && actorSystemFig.getEnabled() ) {
-
- String region = idIoEvent.getRegion();
- if ( region == null ) {
- region = uniqueValuesFig.getAuthoritativeRegion();
- }
- if ( region == null ) {
- region = actorSystemFig.getRegionLocal();
- }
-
- try {
- uniqueValuesService.releaseUniqueValues( applicationScope, entityId, version, region );
- } catch (UniqueValueException e) {
- throw new RuntimeException( "Unable to release unique values for entity " + entityId );
- }
- }
+ // TODO: do we need this or can we rely on UniqueCleanup + Cassandra replication?
+//
+// // actorSystemFig may be null in testing
+// if ( actorSystemFig != null && actorSystemFig.getEnabled() ) {
+//
+// String region = idIoEvent.getRegion();
+// if ( region == null ) {
+// region = uniqueValuesFig.getAuthoritativeRegion();
+// }
+// if ( region == null ) {
+// region = actorSystemFig.getRegionLocal();
+// }
+//
+// try {
+// uniqueValuesService.releaseUniqueValues( applicationScope, entityId, version, region );
+// } catch (UniqueValueException e) {
+// throw new RuntimeException( "Unable to release unique values for entity " + entityId );
+// }
+// }
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d5d89a82/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
index 5deddd6..1104a23 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
@@ -59,15 +59,16 @@ public interface UniqueValuesService extends RouterProducer {
void confirmUniqueValues( ApplicationScope scope, Entity entity, UUID version , String region )
throws UniqueValueException;
+ // TODO: is this really necessary? MarkCommit and UniqueCleanup should do the trick
/**
* Release unique values held by an entity.
- *
* @param scope Application scope of entity.
* @param entityId Id of Entity with unique values to be released
* @param version Version of entity.
* @param region Authoritative Region to be used for this entity or null to use current region.
* @throws UniqueValueException if unique values cannot be reserved.
*/
- void releaseUniqueValues( ApplicationScope scope, Id entityId, UUID version, String region )
- throws UniqueValueException;
+// void releaseUniqueValues( ApplicationScope scope, Id entityId, UUID version, String region )
+// throws UniqueValueException;
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d5d89a82/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
index 4102090..50114be 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
@@ -151,19 +151,23 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
}
- @Override
- public void releaseUniqueValues(ApplicationScope scope, Id entityId, UUID version, String region)
- throws UniqueValueException {
-
- ready();
-
- Iterator<UniqueValue> iterator = table.getUniqueValues( scope, entityId );
-
- while ( iterator.hasNext() ) {
- UniqueValue uniqueValue = iterator.next();
- cancelUniqueField( scope, entityId, uniqueValue.getEntityVersion(), uniqueValue.getField(), region );
- }
- }
+ // TODO: do we need this or can we rely on UniqueCleanup + Cassandra replication?
+
+// @Override
+// public void releaseUniqueValues(ApplicationScope scope, Id entityId, UUID version, String region)
+// throws UniqueValueException {
+//
+// ready();
+//
+// TODO: need to replicate logic from UniqueCleanup and make sure it happens in Authoritative Region
+//
+// Iterator<UniqueValue> iterator = table.getUniqueValues( scope, entityId );
+//
+// while ( iterator.hasNext() ) {
+// UniqueValue uniqueValue = iterator.next();
+// cancelUniqueField( scope, entityId, uniqueValue.getEntityVersion(), uniqueValue.getField(), region );
+// }
+// }
private void reserveUniqueField(
[10/50] usergrid git commit: Merge branch 'release-2.1.1' of
https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/37c482c5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/37c482c5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/37c482c5
Branch: refs/heads/master
Commit: 37c482c5ee4175706a05034e8e3252d03f0b24f7
Parents: c4f72ef 9c4b524
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jun 29 08:45:25 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jun 29 08:45:25 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 40 ++++++++---------
.../asyncevents/EventBuilderImpl.java | 7 ++-
.../UniqueValueSerializationStrategyImpl.java | 11 ++---
.../index/impl/EsEntityIndexImpl.java | 47 +++++++-------------
4 files changed, 43 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
[31/50] usergrid git commit: Add delays to the App Delete tests as UV
cleanup and take a couple of seconds.
Posted by mr...@apache.org.
Add delays to the App Delete tests as UV cleanup and take a couple of seconds.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/db6950b9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/db6950b9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/db6950b9
Branch: refs/heads/master
Commit: db6950b9a476e0f1b77f1333d8852486232e4a9c
Parents: 670248d
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Jul 5 15:16:13 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Jul 5 15:16:13 2016 -0400
----------------------------------------------------------------------
...ApplicationAlreadyExistsExceptionMapper.java | 32 ++++++
.../rest/applications/ApplicationDeleteIT.java | 115 +++++++++++++++++--
2 files changed, 135 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/db6950b9/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ApplicationAlreadyExistsExceptionMapper.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ApplicationAlreadyExistsExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ApplicationAlreadyExistsExceptionMapper.java
new file mode 100644
index 0000000..2c90f5d
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ApplicationAlreadyExistsExceptionMapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.exceptions;
+
+
+import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
+
+import javax.ws.rs.core.Response;
+
+import static javax.ws.rs.core.Response.Status.CONFLICT;
+
+
+public class ApplicationAlreadyExistsExceptionMapper extends AbstractExceptionMapper<ApplicationAlreadyExistsException> {
+ @Override
+ public Response toResponse(ApplicationAlreadyExistsException e ) {
+ return toResponse( CONFLICT, e );
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/db6950b9/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
index 5858289..cc635ed 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
@@ -51,6 +51,55 @@ public class ApplicationDeleteIT extends AbstractRestIT {
public static final int INDEXING_WAIT = 3000;
+ @Test
+ public void testBasicOperation() throws Exception {
+
+ // create app with a collection of "things"
+
+ String orgName = clientSetup.getOrganization().getName();
+ String appToDeleteName = clientSetup.getAppName() + "_appToDelete";
+ Token orgAdminToken = getAdminToken( clientSetup.getUsername(), clientSetup.getUsername());
+
+ List<Entity> entities = new ArrayList<>();
+
+ UUID appToDeleteId = createAppWithCollection(orgName, appToDeleteName, orgAdminToken, entities);
+
+ // delete the app
+
+ clientSetup.getRestClient().management().orgs()
+ .org(orgName).apps().app(appToDeleteId.toString() ).getTarget()
+ .queryParam("access_token", orgAdminToken.getAccessToken() )
+ .queryParam(CONFIRM_APPLICATION_IDENTIFIER, appToDeleteId)
+ .request().delete();
+
+ // test that we can create a new application with the same name
+
+ ApiResponse appCreateAgainResponse = null;
+
+ int retries = 0;
+ while ( retries++ < 100 ) {
+ try {
+ appCreateAgainResponse = clientSetup.getRestClient()
+ .management().orgs().org( orgName ).app().getTarget()
+ .queryParam( "access_token", orgAdminToken.getAccessToken() ).request()
+ .post( javax.ws.rs.client.Entity.json( new Application( appToDeleteName ) ), ApiResponse.class );
+
+ break;
+
+ } catch (Exception e) {
+ logger.error("App not deleted yet. Waiting ... ({})", retries);
+ Thread.sleep( 1000 );
+ }
+ }
+
+ Assert.assertNotNull( appCreateAgainResponse );
+
+ Assert.assertEquals("Must be able to create app with same name as deleted app",
+ (orgName + "/" + appToDeleteName).toLowerCase(),
+ appCreateAgainResponse.getEntities().get(0).get( "name" ));
+ }
+
+
/**
* Test most common use cases.
* <pre>
@@ -62,7 +111,7 @@ public class ApplicationDeleteIT extends AbstractRestIT {
* </pre>
*/
@Test
- public void testBasicOperation() throws Exception {
+ public void testCommonUseCases() throws Exception {
// create app with a collection of "things"
@@ -91,9 +140,11 @@ public class ApplicationDeleteIT extends AbstractRestIT {
.request()
.delete();
+
// test that we can no longer get the app
- try {
+ try { // using /management/orgs/{org-name}/app/{app-name}
+
clientSetup.getRestClient().management().orgs()
.org(orgName).apps().app(appToDeleteName).getTarget()
.queryParam("access_token", orgAdminToken.getAccessToken())
@@ -109,7 +160,8 @@ public class ApplicationDeleteIT extends AbstractRestIT {
}
- try {
+ try { // using /{org-name}/{app-name} path
+
clientSetup.getRestClient().org( orgName ).app( appToDeleteName ).getTarget()
.queryParam( "access_token", orgAdminToken.getAccessToken() ).request()
.get( ApiResponse.class );
@@ -185,11 +237,25 @@ public class ApplicationDeleteIT extends AbstractRestIT {
// test that we can create a new application with the same name
- ApiResponse appCreateAgainResponse = clientSetup.getRestClient()
- .management().orgs().org( orgName ).app().getTarget()
- .queryParam( "access_token", orgAdminToken.getAccessToken() )
- .request()
- .post( javax.ws.rs.client.Entity.json( new Application( appToDeleteName ) ), ApiResponse.class );
+ ApiResponse appCreateAgainResponse = null;
+
+ int retries = 0;
+ while ( retries++ < 20 ) {
+ try {
+ appCreateAgainResponse = clientSetup.getRestClient()
+ .management().orgs().org( orgName ).app().getTarget()
+ .queryParam( "access_token", orgAdminToken.getAccessToken() ).request()
+ .post( javax.ws.rs.client.Entity.json( new Application( appToDeleteName ) ), ApiResponse.class );
+
+ break;
+
+ } catch (Exception e) {
+ logger.error("App not deleted yet. Waiting ... ({})", retries);
+ Thread.sleep( 1000 );
+ }
+ }
+
+ Assert.assertNotNull( appCreateAgainResponse );
Assert.assertEquals("Must be able to create app with same name as deleted app",
(orgName + "/" + appToDeleteName).toLowerCase(),
@@ -312,7 +378,20 @@ public class ApplicationDeleteIT extends AbstractRestIT {
// create new app with same name
- createAppWithCollection(orgName, appToDeleteName, orgAdminToken, entities);
+ UUID newAppId = null;
+ int retries = 0;
+ while ( retries++ < 20 ) {
+ try {
+ newAppId = createAppWithCollection( orgName, appToDeleteName, orgAdminToken, entities );
+ break;
+
+ } catch ( Exception e ) {
+ logger.error("Application not deleted yet. Waiting... ({})", retries);
+ Thread.sleep(1000);
+ }
+ }
+
+ Assert.assertNotNull( newAppId );
// attempt to restore original app, should get 409
@@ -349,12 +428,24 @@ public class ApplicationDeleteIT extends AbstractRestIT {
.orgs().org( orgName ).apps().app( appToDeleteId.toString() ).getTarget()
.queryParam( "access_token", orgAdminToken.getAccessToken() )
.queryParam( CONFIRM_APPLICATION_IDENTIFIER, appToDeleteId )
- .request()
- .delete();
+ .request().delete();
// create new app with same name
- UUID newAppId = createAppWithCollection(orgName, appToDeleteName, orgAdminToken, entities);
+ UUID newAppId = null;
+ int retries = 0;
+ while ( retries++ < 20 ) {
+ try {
+ newAppId = createAppWithCollection( orgName, appToDeleteName, orgAdminToken, entities );
+ break;
+
+ } catch ( Exception e ) {
+ logger.error("Application not deleted yet. Waiting... ({})", retries);
+ Thread.sleep(1000);
+ }
+ }
+
+ Assert.assertNotNull( newAppId );
// attempt to delete new app, it should fail
[30/50] usergrid git commit: Merge branch 'release-2.1.1' of
https://git-wip-us.apache.org/repos/asf/usergrid into usergrid-1268-akka-211
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into usergrid-1268-akka-211
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/670248dd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/670248dd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/670248dd
Branch: refs/heads/master
Commit: 670248ddaff65db78794bbdceb3c569b0543f6d3
Parents: 557ea91 8982ddd
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Jul 5 12:57:24 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Jul 5 12:57:24 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 44 +++++---
.../persistence/collection/FieldSet.java | 12 +++
.../impl/EntityCollectionManagerImpl.java | 27 +----
.../mvcc/stage/write/WriteUniqueVerify.java | 19 +++-
.../serialization/SerializationFig.java | 4 -
.../serialization/impl/MutableFieldSet.java | 12 +++
.../UniqueValueSerializationStrategyImpl.java | 48 +++++++--
.../usergrid/tools/UniqueValueScanner.java | 105 +++++++++++--------
8 files changed, 176 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/670248dd/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/670248dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/670248dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 985137b,01e6621..e7ba967
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@@ -170,10 -121,13 +170,11 @@@ public class WriteUniqueVerify implemen
try {
- // loading will retrieve the oldest unique value entry for the field
// don't use read repair on this pre-write check
- UniqueValueSet set = uniqueValueStrat.load(scope, written.getEntityId().getType(),
- Collections.singletonList(written.getField()), false);
+ // use CL ALL as consistency is extremely important here, more so than performance
+ UniqueValueSet set = uniqueValueStrat.load(scope, ConsistencyLevel.CL_ALL,
+ written.getEntityId().getType(), Collections.singletonList(written.getField()), false);
-
set.forEach(uniqueValue -> {
if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
[36/50] usergrid git commit: Revert "Add support for later versions
of tomcat."
Posted by mr...@apache.org.
Revert "Add support for later versions of tomcat."
This reverts commit 4613e09ee9d5b6df8fae3cc6419142d09a98c312.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/86ae2cd0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/86ae2cd0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/86ae2cd0
Branch: refs/heads/master
Commit: 86ae2cd01175a9f861e4fdcf0fbc0c0e96004334
Parents: 2cae866
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jul 6 22:22:03 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jul 6 22:22:03 2016 -0700
----------------------------------------------------------------------
stack/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/86ae2cd0/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 1bf1f92..dd35891 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -117,7 +117,7 @@
<shiro-version>1.2.4</shiro-version>
<slf4j-version>1.7.2</slf4j-version>
<snakeyaml-version>1.9</snakeyaml-version>
- <tomcat-version>8.5.3</tomcat-version>
+ <tomcat-version>7.0.64</tomcat-version>
<antlr.version>3.4</antlr.version>
<tika.version>1.4</tika.version>
<mockito.version>1.10.8</mockito.version>
[23/50] usergrid git commit: add missing license headers
Posted by mr...@apache.org.
add missing license headers
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1b4e1172
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1b4e1172
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1b4e1172
Branch: refs/heads/master
Commit: 1b4e1172b516eed4746b7a4d7999a329b87b6796
Parents: f0c9fd4
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jul 1 15:17:06 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jul 1 15:17:06 2016 -0400
----------------------------------------------------------------------
.../src/test/resources/usergrid-CHOP.properties | 18 ++++++++++++++++++
.../src/test/resources/usergrid-UNIT.properties | 19 +++++++++++++++++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b4e1172/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
index 78e3400..d404b1e 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
@@ -1,3 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
# These are for CHOP environment settings
cassandra.connections=20
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b4e1172/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
index 44dc399..81a4c17 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
@@ -1,3 +1,22 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
# Keep nothing but overriding test defaults in here
cassandra.connections=50
cassandra.port=9160
[39/50] usergrid git commit: Fixing logic around entity delete to
ensure that UniqueValues and ReservationCache are both cleaned up.
Posted by mr...@apache.org.
Fixing logic around entity delete to ensure that UniqueValues and ReservationCache are both cleaned up.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e6c59ee3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e6c59ee3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e6c59ee3
Branch: refs/heads/master
Commit: e6c59ee3fdc77ce32874ca6b2e0f5fc22a205a20
Parents: 917f0e3
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 12:12:42 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 12:12:42 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../resources/usergrid-custom-test.properties | 13 ++++
.../actorsystem/ActorSystemManagerImpl.java | 5 +-
.../exception/WriteUniqueVerifyException.java | 2 +-
.../mvcc/stage/delete/MarkCommit.java | 34 +++++++-
.../mvcc/stage/write/WriteUniqueVerify.java | 3 +-
.../uniquevalues/ReservationCache.java | 5 ++
.../uniquevalues/ReservationCacheActor.java | 13 +++-
.../uniquevalues/UniqueValueActor.java | 81 +++++++++++---------
.../uniquevalues/UniqueValuesService.java | 15 ++++
.../uniquevalues/UniqueValuesServiceImpl.java | 34 ++++++--
.../uniquevalues/UniqueValuesTable.java | 16 ++--
.../uniquevalues/UniqueValuesTableImpl.java | 8 ++
13 files changed, 170 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 7006f44..41613ac 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2863,7 +2863,7 @@ public class CpEntityManager implements EntityManager {
throws DuplicateUniquePropertyExistsException {
// we may have multiple conflicts, but caller expects only one
- Map<String, Field> violiations = wuve.getVioliations();
+ Map<String, Field> violiations = wuve.getViolations();
if ( violiations != null ) {
Field conflict = violiations.get( violiations.keySet().iterator().next() );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index 0fa9573..dd6612f 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -34,3 +34,16 @@ elasticsearch.buffer_timeout=1
# This property is required to be set and cannot be defaulted anywhere
usergrid.cluster_name=usergrid
+# --- Start: Usergrid cluster/actor system settings
+
+usergrid.cluster.enabled=true
+usergrid.cluster.hostname=localhost
+usergrid.cluster.region.local=us-east
+usergrid.cluster.region.list=us-east
+usergrid.cluster.seeds=us-east\:localhost
+
+collection.uniquevalues.actors=300
+collection.uniquevalues.authoritative.region=us-east
+
+# --- End: Usergrid cluster/actor system settings
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index 099b875..8399979 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -282,12 +282,13 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
String region = currentRegion;
List<String> seeds = getSeedsByRegion().get( region );
- int lastColon = seeds.get(0).lastIndexOf(":") + 1;
- final Integer regionPort = Integer.parseInt( seeds.get(0).substring( lastColon ));
logger.info( "Akka Config for region {} is:\n" + " Hostname {}\n" + " Seeds {}\n",
region, hostname, seeds );
+ int lastColon = seeds.get(0).lastIndexOf(":") + 1;
+ final Integer regionPort = Integer.parseInt( seeds.get(0).substring( lastColon ));
+
Map<String, Object> configMap = new HashMap<String, Object>() {{
put( "akka", new HashMap<String, Object>() {{
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
index 81165fa..a223a9e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
@@ -40,7 +40,7 @@ public class WriteUniqueVerifyException extends CollectionRuntimeException {
/**
* Get map of Fields in violation, keyed by field name.
*/
- public Map<String, Field> getVioliations() {
+ public Map<String, Field> getViolations() {
return violations;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index 460c271..70c785e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@ -21,6 +21,10 @@ package org.apache.usergrid.persistence.collection.mvcc.stage.delete;
import java.util.UUID;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueException;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesFig;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,12 +64,19 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
private final SerializationFig serializationFig;
private final UniqueValueSerializationStrategy uniqueValueStrat;
private final Keyspace keyspace;
+ private final ActorSystemFig actorSystemFig;
+ private final UniqueValuesFig uniqueValuesFig;
+ private final UniqueValuesService uniqueValuesService;
@Inject
public MarkCommit( final MvccLogEntrySerializationStrategy logStrat,
final MvccEntitySerializationStrategy entityStrat,
- final UniqueValueSerializationStrategy uniqueValueStrat, final SerializationFig serializationFig,
+ final UniqueValueSerializationStrategy uniqueValueStrat,
+ final SerializationFig serializationFig,
+ final ActorSystemFig actorSystemFig,
+ final UniqueValuesFig uniqueValuesFig,
+ final UniqueValuesService uniqueValuesService,
final Keyspace keyspace ) {
@@ -76,6 +87,9 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
this.entityStrat = entityStrat;
this.serializationFig = serializationFig;
this.uniqueValueStrat = uniqueValueStrat;
+ this.actorSystemFig = actorSystemFig;
+ this.uniqueValuesFig = uniqueValuesFig;
+ this.uniqueValuesService = uniqueValuesService;
this.keyspace = keyspace;
}
@@ -111,6 +125,24 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to mark entry as deleted" );
}
+
+ // actorSystemFig may be null in testing
+ if ( actorSystemFig != null && actorSystemFig.getEnabled() ) {
+
+ String region = idIoEvent.getRegion();
+ if ( region == null ) {
+ region = uniqueValuesFig.getAuthoritativeRegion();
+ }
+ if ( region == null ) {
+ region = actorSystemFig.getRegionLocal();
+ }
+
+ try {
+ uniqueValuesService.releaseUniqueValues( applicationScope, entityId, version, region );
+ } catch (UniqueValueException e) {
+ throw new RuntimeException( "Unable to release unique values for entity " + entityId );
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index e7ba967..1f86440 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -219,8 +219,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
logger.trace("Pre-write unique violations found, raising exception before executing first write");
}
- throw new WriteUniqueVerifyException(mvccEntity, scope,
- preWriteUniquenessViolations );
+ throw new WriteUniqueVerifyException(mvccEntity, scope, preWriteUniquenessViolations );
}
//short circuit nothing to do
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java
index f1e68b2..8e2e51e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java
@@ -72,6 +72,11 @@ public class ReservationCache {
cache.invalidate( cancellation.getConsistentHashKey() );
}
+ public void cancelReservation( UniqueValueActor.Response response ) {
+ if ( ttl == 0 ) { return; }
+ cache.invalidate( response.getConsistentHashKey() );
+ }
+
public CacheStats getStats() {
return cache.stats();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
index 51f5c8c..158b099 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
@@ -54,12 +54,19 @@ public class ReservationCacheActor extends UntypedActor {
}
} else if ( msg instanceof UniqueValueActor.Cancellation ) {
- UniqueValueActor.Cancellation can = (UniqueValueActor.Cancellation)msg;
+ UniqueValueActor.Cancellation can = (UniqueValueActor.Cancellation) msg;
ReservationCache.getInstance().cancelReservation( can );
- if ( ++cancellationCount % 10 == 0 ) {
- logger.info("Received {} cancellations", cancellationCount);
+ if (++cancellationCount % 10 == 0) {
+ logger.info( "Received {} cancellations", cancellationCount );
}
+ logger.debug("Removing cancelled {} from reservation cache", can.getConsistentHashKey());
+
+ } else if ( msg instanceof UniqueValueActor.Response ) {
+ UniqueValueActor.Response response = (UniqueValueActor.Response) msg;
+ ReservationCache.getInstance().cancelReservation( response );
+
+ logger.info("Removing completed {} from reservation cache", response.getConsistentHashKey());
} else if (msg instanceof DistributedPubSubMediator.SubscribeAck) {
logger.debug( "subscribing" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
index a14c63e..501037f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
@@ -16,10 +16,7 @@
*/
package org.apache.usergrid.persistence.collection.uniquevalues;
-import akka.actor.ActorRef;
import akka.actor.UntypedActor;
-import akka.cluster.pubsub.DistributedPubSub;
-import akka.cluster.pubsub.DistributedPubSubMediator;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -51,8 +48,6 @@ public class UniqueValueActor extends UntypedActor {
// TODO: is there a way to avoid this ugly kludge? see also: ClusterSingletonRouter
this.table = UniqueValuesServiceImpl.injector.getInstance( UniqueValuesTable.class );
this.actorSystemManager = UniqueValuesServiceImpl.injector.getInstance( ActorSystemManager.class );
-
- //logger.info("UniqueValueActor {} is live with table {}", name, table);
}
@Override
@@ -77,24 +72,27 @@ public class UniqueValueActor extends UntypedActor {
if ( owner != null && owner.equals( res.getOwner() )) {
// sender already owns this unique value
- getSender().tell( new Response( Response.Status.IS_UNIQUE ), getSender() );
+ getSender().tell( new Response( Response.Status.IS_UNIQUE, res.getConsistentHashKey() ),
+ getSender() );
return;
} else if ( owner != null && !owner.equals( res.getOwner() )) {
// tell sender value is not unique
- getSender().tell( new Response( Response.Status.NOT_UNIQUE ), getSender() );
+ getSender().tell( new Response( Response.Status.NOT_UNIQUE, res.getConsistentHashKey() ),
+ getSender() );
return;
}
table.reserve( res.getApplicationScope(), res.getOwner(), res.getOwnerVersion(), res.getField() );
- getSender().tell( new Response( Response.Status.IS_UNIQUE ), getSender() );
+ getSender().tell( new Response( Response.Status.IS_UNIQUE, res.getConsistentHashKey() ),
+ getSender() );
actorSystemManager.publishToAllRegions( "content", new Reservation( res ), getSelf() );
} catch (Throwable t) {
- getSender().tell( new Response( Response.Status.ERROR ), getSender() );
+ getSender().tell( new Response( Response.Status.ERROR, res.getConsistentHashKey() ), getSender() );
logger.error( "Error processing request", t );
@@ -112,23 +110,29 @@ public class UniqueValueActor extends UntypedActor {
if ( owner != null && !owner.equals( con.getOwner() )) {
// cannot reserve, somebody else owns the unique value
- getSender().tell( new Response( Response.Status.NOT_UNIQUE ), getSender() );
+ Response response = new Response( Response.Status.NOT_UNIQUE, con.getConsistentHashKey());
+ getSender().tell( response, getSender() );
+ actorSystemManager.publishToAllRegions( "content", response, getSelf() );
return;
} else if ( owner == null ) {
// cannot commit without first reserving
- getSender().tell( new Response( Response.Status.BAD_REQUEST ), getSender() );
+ Response response = new Response( Response.Status.BAD_REQUEST, con.getConsistentHashKey());
+ getSender().tell( response, getSender() );
+ actorSystemManager.publishToAllRegions( "content", response, getSelf() );
return;
}
table.confirm( con.getApplicationScope(), con.getOwner(), con.getOwnerVersion(), con.getField() );
- getSender().tell( new Response( Response.Status.IS_UNIQUE ), getSender() );
+ Response response = new Response( Response.Status.IS_UNIQUE, con.getConsistentHashKey() );
+ getSender().tell( response, getSender() );
- actorSystemManager.publishToAllRegions( "content", new Reservation( con ), getSelf() );
+ actorSystemManager.publishToAllRegions( "content", response, getSelf() );
} catch (Throwable t) {
- getSender().tell( new Response( Response.Status.ERROR ), getSender() );
+ getSender().tell( new Response( Response.Status.ERROR, con.getConsistentHashKey() ),
+ getSender() );
logger.error( "Error processing request", t );
} finally {
@@ -144,23 +148,34 @@ public class UniqueValueActor extends UntypedActor {
if ( owner != null && !owner.equals( can.getOwner() )) {
// cannot cancel, somebody else owns the unique value
- getSender().tell( new Response( Response.Status.NOT_UNIQUE ), getSender() );
+ getSender().tell( new Response( Response.Status.NOT_UNIQUE, can.getConsistentHashKey() ),
+ getSender() );
return;
} else if ( owner == null ) {
+
// cannot cancel unique value that does not exist
- getSender().tell( new Response( Response.Status.BAD_REQUEST ), getSender() );
+ getSender().tell( new Response( Response.Status.BAD_REQUEST, can.getConsistentHashKey() ),
+ getSender() );
+
+ // unique value record may have already been cleaned up, also clear cache
+ actorSystemManager.publishToAllRegions( "content", new Cancellation( can ), getSelf() );
+
return;
}
- table.confirm( can.getApplicationScope(), can.getOwner(), can.getOwnerVersion(), can.getField() );
+ table.cancel( can.getApplicationScope(), can.getOwner(), can.getOwnerVersion(), can.getField() );
+
+ logger.debug("Removing {} from unique values table", can.getConsistentHashKey());
- getSender().tell( new Response( Response.Status.SUCCESS ), getSender() );
+ getSender().tell( new Response( Response.Status.SUCCESS, can.getConsistentHashKey() ),
+ getSender() );
- actorSystemManager.publishToAllRegions( "content", new Reservation( can ), getSelf() );
+ actorSystemManager.publishToAllRegions( "content", new Cancellation( can ), getSelf() );
} catch (Throwable t) {
- getSender().tell( new Response( Response.Status.ERROR ), getSender() );
+ getSender().tell( new Response( Response.Status.ERROR, can.getConsistentHashKey() ),
+ getSender() );
logger.error( "Error processing request", t );
}
@@ -188,7 +203,7 @@ public class UniqueValueActor extends UntypedActor {
this.field = field;
StringBuilder sb = new StringBuilder();
- sb.append( applicationScope.getApplication() );
+ sb.append( applicationScope.getApplication().getUuid() );
sb.append(":");
sb.append( owner.getType() );
sb.append(":");
@@ -198,22 +213,7 @@ public class UniqueValueActor extends UntypedActor {
this.consistentHashKey = sb.toString();
}
public Request( Request req ) {
-
- this.applicationScope = req.applicationScope;
- this.owner = req.owner;
- this.ownerVersion = req.ownerVersion;
- this.field = req.field;
-
- StringBuilder sb = new StringBuilder();
- sb.append( req.applicationScope.getApplication() );
- sb.append(":");
- sb.append( req.owner.getType() );
- sb.append(":");
- sb.append( req.field.getName() );
- sb.append(":");
- sb.append( req.field.getValue().toString() );
- this.consistentHashKey = sb.toString();
-
+ this( req.getApplicationScope(), req.getOwner(), req.getOwnerVersion(), req.getField() );
}
public ApplicationScope getApplicationScope() {
return applicationScope;
@@ -238,13 +238,18 @@ public class UniqueValueActor extends UntypedActor {
public static class Response implements Serializable {
public enum Status { IS_UNIQUE, NOT_UNIQUE, SUCCESS, ERROR, BAD_REQUEST }
final Status status;
+ final String consistentHashKey;
- public Response(Status status) {
+ public Response(Status status, String consistentHashKey ) {
this.status = status;
+ this.consistentHashKey = consistentHashKey;
}
public Status getStatus() {
return status;
}
+ public String getConsistentHashKey() {
+ return consistentHashKey;
+ }
}
public static class Reservation extends Request implements Serializable {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
index e9a9f50..5deddd6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
@@ -20,8 +20,11 @@ package org.apache.usergrid.persistence.collection.uniquevalues;
import org.apache.usergrid.persistence.actorsystem.RouterProducer;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.Field;
import java.util.UUID;
@@ -55,4 +58,16 @@ public interface UniqueValuesService extends RouterProducer {
*/
void confirmUniqueValues( ApplicationScope scope, Entity entity, UUID version , String region )
throws UniqueValueException;
+
+ /**
+ * Release unique values held by an entity.
+ *
+ * @param scope Application scope of entity.
+ * @param entityId Id of Entity with unique values to be released
+ * @param version Version of entity.
+ * @param region Authoritative Region to be used for this entity or null to use current region.
+ * @throws UniqueValueException if unique values cannot be reserved.
+ */
+ void releaseUniqueValues( ApplicationScope scope, Id entityId, UUID version, String region )
+ throws UniqueValueException;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
index fcee8fa..4102090 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
@@ -35,8 +35,10 @@ import com.google.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.actorsystem.GuiceActorProducer;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,6 +46,7 @@ import scala.concurrent.Await;
import scala.concurrent.Future;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -109,13 +112,12 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
try {
cancelUniqueField( scope, entity, version, field, region );
} catch (Throwable ignored) {
- logger.debug( "Error canceling unique field", ignored );
+ logger.error( "Error canceling unique field", ignored );
}
}
}
throw e;
}
-
}
@@ -149,6 +151,21 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
}
+ @Override
+ public void releaseUniqueValues(ApplicationScope scope, Id entityId, UUID version, String region)
+ throws UniqueValueException {
+
+ ready();
+
+ Iterator<UniqueValue> iterator = table.getUniqueValues( scope, entityId );
+
+ while ( iterator.hasNext() ) {
+ UniqueValue uniqueValue = iterator.next();
+ cancelUniqueField( scope, entityId, uniqueValue.getEntityVersion(), uniqueValue.getField(), region );
+ }
+ }
+
+
private void reserveUniqueField(
ApplicationScope scope, Entity entity, UUID version, Field field, String region ) throws UniqueValueException {
@@ -177,11 +194,18 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
}
- private void cancelUniqueField(
- ApplicationScope scope, Entity entity, UUID version, Field field, String region ) throws UniqueValueException {
+ private void cancelUniqueField( ApplicationScope scope,
+ Entity entity, UUID version, Field field, String region ) throws UniqueValueException {
+
+ cancelUniqueField( scope, entity.getId(), version, field, region );
+ }
+
+
+ private void cancelUniqueField( ApplicationScope scope,
+ Id entityId, UUID version, Field field, String region ) throws UniqueValueException {
UniqueValueActor.Cancellation request = new UniqueValueActor.Cancellation(
- scope, entity.getId(), version, field );
+ scope, entityId, version, field );
if ( actorSystemManager.getCurrentRegion().equals( region ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java
index 0e69ef7..1bb2534 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java
@@ -19,24 +19,24 @@
package org.apache.usergrid.persistence.collection.uniquevalues;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
+import java.util.Iterator;
import java.util.UUID;
public interface UniqueValuesTable {
- Id lookupOwner(
- ApplicationScope applicationScope, String type, Field field ) throws ConnectionException;
+ Id lookupOwner( ApplicationScope scope, String type, Field field ) throws ConnectionException;
- void reserve(
- ApplicationScope applicationScope, Id owner, UUID version, Field field ) throws ConnectionException;
+ void reserve( ApplicationScope scope, Id owner, UUID version, Field field ) throws ConnectionException;
- void confirm(
- ApplicationScope applicationScope, Id owner, UUID version, Field field ) throws ConnectionException;
+ void confirm( ApplicationScope scope, Id owner, UUID version, Field field ) throws ConnectionException;
- void cancel(
- ApplicationScope applicationScope, Id owner, UUID version, Field field ) throws ConnectionException;
+ void cancel( ApplicationScope scope, Id owner, UUID version, Field field ) throws ConnectionException;
+
+ Iterator<UniqueValue> getUniqueValues(ApplicationScope scope, Id entityId );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java
index 9cb13be..2cad32c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java
@@ -33,7 +33,9 @@ import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
import java.util.UUID;
@@ -83,4 +85,10 @@ public class UniqueValuesTableImpl implements UniqueValuesTable {
final MutationBatch write = strat.delete( scope, uv );
write.execute();
}
+
+ @Override
+ public Iterator<UniqueValue> getUniqueValues(ApplicationScope scope, Id entityId) {
+ return strat.getAllUniqueFields( scope, entityId );
+ }
+
}
[37/50] usergrid git commit: Fixes to tests and test properties.
Posted by mr...@apache.org.
Fixes to tests and test properties.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/996bf09f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/996bf09f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/996bf09f
Branch: refs/heads/master
Commit: 996bf09fbc33bba3c27a26143880ceb100a38936
Parents: db6950b
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 12:09:43 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 12:09:43 2016 -0400
----------------------------------------------------------------------
.../src/test/resources/usergrid-test.properties | 15 ++
.../corepersistence/index/IndexServiceTest.java | 70 ++++---
.../collection/EntityCollectionManagerIT.java | 45 +++--
.../mvcc/stage/delete/MarkCommitTest.java | 3 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 2 +-
.../UniqueValuesServiceDeleteTest.java | 185 +++++++++++++++++++
.../src/test/resources/usergrid.properties | 3 +
.../org/apache/usergrid/rest/UniqueCatsIT.java | 2 +-
.../rest/applications/ApplicationDeleteIT.java | 2 +-
9 files changed, 282 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/config/src/test/resources/usergrid-test.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/test/resources/usergrid-test.properties b/stack/config/src/test/resources/usergrid-test.properties
index 29bb332..368c585 100644
--- a/stack/config/src/test/resources/usergrid-test.properties
+++ b/stack/config/src/test/resources/usergrid-test.properties
@@ -83,6 +83,21 @@ usergrid.version.properties=1.0.0
# build number for display
usergrid.version.build=0.1
+
+# --- Start: Usergrid cluster/actor system settings
+
+usergrid.cluster.enabled=true
+usergrid.cluster.hostname=localhost
+usergrid.cluster.region.local=us-east
+usergrid.cluster.region.list=us-east
+usergrid.cluster.seeds=us-east\:localhost
+
+collection.uniquevalues.actors=300
+collection.uniquevalues.authoritative.region=us-east
+
+# --- End: Usergrid cluster/actor system settings
+
+
usergrid.counter.batch.interval=10
#usergrid.auth.token_secret_salt=super secret token value
#usergrid.auth.token_expires_from_last_use=false
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
index dab14ed..fba6e7c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
@@ -20,62 +20,54 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.graph.MarkedEdge;
-import org.apache.usergrid.persistence.index.impl.IndexProducer;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
+import com.google.inject.Inject;
+import net.jcip.annotations.NotThreadSafe;
import org.apache.usergrid.corepersistence.TestIndexModule;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueActor;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.CandidateResults;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.impl.EsRunner;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.index.impl.IndexOperation;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.google.inject.Inject;
-
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
import rx.Observable;
import rx.schedulers.Schedulers;
+import java.util.*;
+
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionEdge;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicationScope;
import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
@RunWith( EsRunner.class )
@UseModules( { TestIndexModule.class } )
+@NotThreadSafe//anything that changes the system version state is not safe to be run concurrently
public class IndexServiceTest {
@Inject
public IndexService indexService;
-
@Inject
public GraphManagerFactory graphManagerFactory;
@@ -98,12 +90,39 @@ public class IndexServiceTest {
public ApplicationScope applicationScope;
+ @Inject
+ ActorSystemManager actorSystemManager;
+
+ @Inject
+ UniqueValuesService uniqueValuesService;
+
+
+ private static Map<Integer, Boolean> startedAkka = new HashMap<>();
+
+
+ protected synchronized void initAkka(
+ int port, ActorSystemManager actorSystemManager, UniqueValuesService uniqueValuesService ) {
+
+ if ( startedAkka.get(port) == null ) {
+
+ actorSystemManager.registerRouterProducer( uniqueValuesService );
+ actorSystemManager.registerMessageType( UniqueValueActor.Request.class, "/user/uvProxy" );
+ actorSystemManager.registerMessageType( UniqueValueActor.Reservation.class, "/user/uvProxy" );
+ actorSystemManager.registerMessageType( UniqueValueActor.Cancellation.class, "/user/uvProxy" );
+ actorSystemManager.registerMessageType( UniqueValueActor.Confirmation.class, "/user/uvProxy" );
+ actorSystemManager.start( "localhost", port, "us-east" );
+ actorSystemManager.waitForClientActor();
+
+ startedAkka.put( port, true );
+ }
+ }
+
@Before
public void setup() {
applicationScope = getApplicationScope( UUIDGenerator.newTimeUUID() );
-
graphManager = graphManagerFactory.createEdgeManager( applicationScope );
+ initAkka( 2555, actorSystemManager, uniqueValuesService );
}
@@ -140,7 +159,6 @@ public class IndexServiceTest {
@Test( )
public void testSingleCollectionConnection() throws InterruptedException {
-
ApplicationScope applicationScope =
new ApplicationScopeImpl( new SimpleId( UUID.randomUUID(), "application" ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index ceda1a8..dfff51c 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -49,6 +49,8 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import rx.Observable;
import java.util.ArrayList;
@@ -63,6 +65,7 @@ import static org.junit.Assert.*;
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
public class EntityCollectionManagerIT extends AbstractUniqueValueTest {
+ private static final Logger logger = LoggerFactory.getLogger( EntityCollectionManagerIT.class );
@Inject
private EntityCollectionManagerFactory factory;
@@ -137,7 +140,7 @@ public class EntityCollectionManagerIT extends AbstractUniqueValueTest {
}
catch ( Exception ex ) {
WriteUniqueVerifyException e = ( WriteUniqueVerifyException ) ex;
- assertEquals( 1, e.getVioliations().size() );
+ assertEquals( 1, e.getViolations().size() );
}
}
}
@@ -319,66 +322,78 @@ public class EntityCollectionManagerIT extends AbstractUniqueValueTest {
@Test
public void writeAndGetField2X() throws InterruptedException {
- ApplicationScope collectionScope1 = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ // create entity with unique testField
+ ApplicationScope collectionScope1 = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final Id entityId = new SimpleId( "test" );
Entity firstInstance = new Entity( entityId );
Field firstField = new StringField( "testField", "unique", true );
firstInstance.setField( firstField );
EntityCollectionManager manager = factory.createCollectionManager( collectionScope1 );
-
Observable<Entity> observable = manager.write( firstInstance, null );
-
Entity createReturned = observable.toBlocking().lastOrDefault( null );
-
assertNotNull( "Id was assigned", createReturned.getId() );
assertNotNull( "Version was assigned", createReturned.getVersion() );
+ // get entity via that unique field, should get correct entity
+
final Id existingId = manager.getIdField( firstInstance.getId().getType(), firstField )
.toBlocking().lastOrDefault( null );
assertNotNull( existingId );
assertEquals( firstInstance.getId(), existingId );
+ // get entity via bogus unique field that does not exist, should get null
+
Field fieldNull = new StringField( "testFieldNotThere", "uniquely", true );
final Id noId = manager.getIdField( firstInstance.getId().getType(), fieldNull )
.toBlocking().lastOrDefault( null );
assertNull( noId );
+ // ensure we clean up
- //ensure we clean up
+ // set a different unique field to the entity we created above
+ // this should effectively remove the original unique testField that we created above
Entity secondInstance = new Entity( entityId );
Field secondField = new StringField( firstField.getName(), "unique2", true );
secondInstance.setField( secondField );
Observable<Entity> observableSecond = manager.write( secondInstance, null );
-
Entity createReturnedSecond = observableSecond.toBlocking().lastOrDefault( null );
-
assertNotNull( "Id was assigned", createReturnedSecond.getId() );
assertNotNull( "Version was assigned", createReturnedSecond.getVersion() );
assertNotEquals( "Versions should not be equal",
createReturned.getVersion(), createReturnedSecond.getVersion() );
- //sanity check, get the entity to ensure it's the right version
+ // sanity check, get the entity to ensure it's the right version
final Entity loadedVersion = manager.load( entityId ).toBlocking().last();
assertEquals(entityId, loadedVersion.getId());
assertEquals(createReturnedSecond.getVersion(), loadedVersion.getVersion());
- //give clean time to run. need to finish the todo below
- Thread.sleep( 2000 );
+ // give clean time to run. need to finish the todo below
- //TODO, we need to implement verify and repair on this
- final Id idFirst = manager.getIdField( firstInstance.getId().getType(), firstField )
- .toBlocking().lastOrDefault( null );
- assertNull(idFirst);
+ Id idFirst = null;
+ int retries = 0;
+ while ( retries++ < 20 ) {
+
+ //TODO, we need to implement verify and repair on this
+ idFirst = manager.getIdField( firstInstance.getId().getType(), firstField )
+ .toBlocking().lastOrDefault( null );
+ if ( idFirst == null ) {
+ break;
+ }
+
+ logger.error("Clean no run yet, waiting ({})", retries);
+ Thread.sleep( 2000 );
+ }
+ assertNull(idFirst);
final Id idSecond = manager.getIdField( secondInstance.getId().getType(), secondField )
.toBlocking().lastOrDefault( null );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
index a0ee6be..89169ac 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
@@ -120,7 +120,8 @@ public class MarkCommitTest extends AbstractMvccEntityStageTest {
.thenReturn( entityMutation );
- new MarkCommit( logStrategy, mvccEntityStrategy, uniqueValueSerializationStrategy, serializationFig, keyspace ).call( event );
+ new MarkCommit( logStrategy, mvccEntityStrategy, uniqueValueSerializationStrategy, serializationFig,
+ null, null, null, keyspace ).call( event );
//TODO: This doesn't assert anything, this needs fixed (should be a fail technically)
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
index c70cd1a..401d23e 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
@@ -123,7 +123,7 @@ public class WriteUniqueVerifyIT extends AbstractUniqueValueTest {
} catch ( Exception ex ) {
WriteUniqueVerifyException e = (WriteUniqueVerifyException)ex;
- assertTrue( !e.getVioliations().isEmpty() );
+ assertTrue( !e.getViolations().isEmpty() );
}
// ensure we can update original entity without error
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
new file mode 100644
index 0000000..3f58834
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.collection.uniquevalues;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
+import org.apache.usergrid.persistence.collection.AbstractUniqueValueTest;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import rx.Observable;
+
+import java.util.Collection;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+
+/**
+ * Test the unique values service.
+ */
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
+public class UniqueValuesServiceDeleteTest extends AbstractUniqueValueTest {
+ private static final Logger logger = LoggerFactory.getLogger( UniqueValuesServiceDeleteTest.class );
+
+ @Inject
+ private EntityCollectionManagerFactory factory;
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+ @Inject
+ ActorSystemFig actorSystemFig;
+
+ @Inject
+ ActorSystemManager actorSystemManager;
+
+ @Inject
+ UniqueValuesService uniqueValuesService;
+
+
+ int numThreads = 6;
+ int poolSize = 5;
+ int numUsers = 100;
+
+
+ @Before
+ public void initAkka() {
+ // each test class needs unique port number
+ initAkka( 2555, actorSystemManager, uniqueValuesService );
+ }
+
+
+ /**
+ * Use multiple threads to attempt to create entities with duplicate usernames.
+ */
+ @Test
+ public void testDuplicatePrevention() throws Exception {
+
+ initAkka();
+
+ final AtomicInteger successCounter = new AtomicInteger( 0 );
+ final AtomicInteger errorCounter = new AtomicInteger( 0 );
+
+ Multimap<String, Entity> usersCreated =
+ generateDuplicateUsers( numUsers, successCounter, errorCounter );
+
+ int userCount = 0;
+ int usernamesWithDuplicates = 0;
+ for ( String username : usersCreated.keySet() ) {
+ Collection<Entity> users = usersCreated.get( username );
+ if ( users.size() > 1 ) {
+ usernamesWithDuplicates++;
+ }
+ userCount++;
+ }
+
+ Assert.assertEquals( 0, usernamesWithDuplicates );
+
+ Assert.assertEquals( numUsers, successCounter.get() );
+ Assert.assertEquals( 0, errorCounter.get() );
+ Assert.assertEquals( numUsers, usersCreated.size() );
+ Assert.assertEquals( numUsers, userCount );
+ }
+
+
+ private Multimap<String, Entity> generateDuplicateUsers(
+ int numUsers, AtomicInteger successCounter, AtomicInteger errorCounter ) {
+
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+
+ EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ Multimap<String, Entity> usersCreated =
+ Multimaps.synchronizedListMultimap( ArrayListMultimap.create() );
+
+ ExecutorService execService = Executors.newFixedThreadPool( poolSize );
+
+ for (int i = 0; i < numUsers; i++) {
+
+ // multiple threads simultaneously trying to create a user with the same propertyName
+ for (int j = 0; j < numThreads; j++) {
+ String username = "user_" + i;
+
+ execService.submit( () -> {
+
+ try {
+
+ // give entity two unqiue fields username and email
+ Entity newEntity = new Entity( new SimpleId( "user" ) );
+ newEntity.setField( new StringField( "username", username, true ) );
+ newEntity.setField( new StringField( "email", username + "@example.org", true ) );
+
+ Observable<Entity> observable = manager.write( newEntity, null );
+ Entity returned = observable.toBlocking().lastOrDefault( null );
+
+ usersCreated.put( username, newEntity );
+ successCounter.incrementAndGet();
+
+ logger.debug("Created user {}", username);
+
+ } catch ( Throwable t ) {
+ if ( t instanceof WriteUniqueVerifyException) {
+ // we expect lots of these
+ } else {
+ errorCounter.incrementAndGet();
+ logger.error( "Error creating user " + username, t );
+ }
+ }
+
+ } );
+ }
+ }
+ execService.shutdown();
+
+ try {
+ while (!execService.awaitTermination( 60, TimeUnit.SECONDS )) {
+ System.out.println( "Waiting..." );
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return usersCreated;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid.properties b/stack/corepersistence/collection/src/test/resources/usergrid.properties
index ebc2ad4..8d615d0 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid.properties
@@ -20,6 +20,7 @@
# This property is required to be set and cannot be defaulted anywhere
usergrid.cluster_name=usergrid
+# --- Start: Usergrid cluster/actor system settings
usergrid.cluster.enabled=true
usergrid.cluster.hostname=localhost
@@ -29,3 +30,5 @@ usergrid.cluster.seeds=us-east\:localhost
collection.uniquevalues.actors=300
collection.uniquevalues.authoritative.region=us-east
+
+# --- End: Usergrid cluster/actor system settings
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
index 0120660..c0209af 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
@@ -49,7 +49,7 @@ import static com.codahale.metrics.MetricRegistry.name;
/**
- * Tests that Catgrid will not allow creation of entities with duplicate names.
+ * Tests that Usergrid will not allow creation of entities with duplicate names.
*
* Intended for use against a production-like cluster, not run during normal JUnit testing.
*
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
index cc635ed..c1301bb 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
@@ -77,7 +77,7 @@ public class ApplicationDeleteIT extends AbstractRestIT {
ApiResponse appCreateAgainResponse = null;
int retries = 0;
- while ( retries++ < 100 ) {
+ while ( retries++ < 30 ) {
try {
appCreateAgainResponse = clientSetup.getRestClient()
.management().orgs().org( orgName ).app().getTarget()
[16/50] usergrid git commit: Expose option to perform the
uniqueIndexRepair higher up to the callers. This allows better control of
when the repair happens on the write path.
Posted by mr...@apache.org.
Expose option to perform the uniqueIndexRepair higher up to the callers. This allows better control of when the repair happens on the write path.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/82428785
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/82428785
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/82428785
Branch: refs/heads/master
Commit: 824287853137334cca134efbcfc61e6c797e7e0e
Parents: a6c307b
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jun 29 23:03:02 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jun 29 23:03:02 2016 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/CpEntityManager.java | 9 +++++----
.../usergrid/persistence/EntityManager.java | 4 ++--
.../apache/usergrid/persistence/PathQuery.java | 3 ++-
.../impl/EntityCollectionManagerImpl.java | 8 +++++---
.../mvcc/stage/write/WriteUniqueVerify.java | 18 +++++++++++-------
.../UniqueValueSerializationStrategy.java | 18 +++++++++++++++++-
.../UniqueValueSerializationStrategyImpl.java | 8 ++++++++
...UniqueValueSerializationStrategyProxyImpl.java | 13 +++++++++++++
.../services/AbstractCollectionService.java | 9 ++++++---
.../services/AbstractConnectionsService.java | 5 ++++-
10 files changed, 73 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 3dc0d13..1ed05bd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -853,7 +853,7 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public Entity getUniqueEntityFromAlias(String collectionType, String aliasType, boolean useReadRepair){
+ public Entity getUniqueEntityFromAlias(String collectionType, String aliasType, boolean uniqueIndexRepair){
String collName = Schema.defaultCollectionName( collectionType );
String propertyName = Schema.getDefaultSchema().aliasProperty( collName );
@@ -866,7 +866,7 @@ public class CpEntityManager implements EntityManager {
StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString());
Observable<FieldSet> fieldSetObservable = ecm.getEntitiesFromFields(
- Inflector.getInstance().singularize( collectionType ), Arrays.<Field>asList( uniqueLookupRepairField ), useReadRepair);
+ Inflector.getInstance().singularize( collectionType ), Arrays.<Field>asList( uniqueLookupRepairField ), uniqueIndexRepair);
if(fieldSetObservable == null){
@@ -893,14 +893,15 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public UUID getUniqueIdFromAlias( String collectionType, String aliasType ){
+ public UUID getUniqueIdFromAlias(String collectionType, String aliasType, boolean uniqueIndexRepair){
String collName = Schema.defaultCollectionName( collectionType );
String propertyName = Schema.getDefaultSchema().aliasProperty( collName );
StringField uniqueLookupRepairField = new StringField( propertyName, aliasType);
Observable<FieldSet> fieldSetObservable = ecm.getEntitiesFromFields(
- Inflector.getInstance().singularize( collectionType ), Collections.singletonList(uniqueLookupRepairField), true);
+ Inflector.getInstance().singularize( collectionType ),
+ Collections.singletonList(uniqueLookupRepairField), uniqueIndexRepair);
if(fieldSetObservable == null){
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 7e25a80..874e618 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -721,9 +721,9 @@ public interface EntityManager {
public void flushManagerCaches();
- public Entity getUniqueEntityFromAlias(String aliasType, String aliasValue, boolean useReadRepair);
+ public Entity getUniqueEntityFromAlias(String aliasType, String aliasValue, boolean uniqueIndexRepair);
- public UUID getUniqueIdFromAlias( String aliasType, String aliasValue );
+ public UUID getUniqueIdFromAlias(String aliasType, String aliasValue, boolean uniqueIndexRepair);
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
index 30636ab..c5833af 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
@@ -128,7 +128,8 @@ public class PathQuery<E> {
String name = query.getSingleNameOrEmailIdentifier();
String entityType = InflectionUtils.singularize(query.getCollection());
- UUID entityId = em.getUniqueIdFromAlias( entityType, name );
+ // don't use unique index repair on read only logic
+ UUID entityId = em.getUniqueIdFromAlias( entityType, name, false);
if( entityId == null){
throw new
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 523b4df..658c68c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -325,14 +325,16 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
* Retrieves all entities that correspond to each field given in the Collection.
*/
@Override
- public Observable<FieldSet> getEntitiesFromFields(final String type, final Collection<Field> fields, boolean useReadRepair) {
+ public Observable<FieldSet> getEntitiesFromFields(final String type, final Collection<Field> fields,
+ boolean uniqueIndexRepair) {
final Observable<FieldSet> fieldSetObservable = Observable.just( fields ).map( fields1 -> {
try {
final UUID startTime = UUIDGenerator.newTimeUUID();
//Get back set of unique values that correspond to collection of fields
- UniqueValueSet set = uniqueValueSerializationStrategy.load( applicationScope, type, fields1 );
+ UniqueValueSet set =
+ uniqueValueSerializationStrategy.load( applicationScope, type, fields1 , uniqueIndexRepair);
//Short circuit if we don't have any uniqueValues from the given fields.
if ( !set.iterator().hasNext() ) {
@@ -407,7 +409,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
response.addEntity( expectedUnique.getField(), entity );
}
- if ( useReadRepair && deleteBatch.getRowCount() > 0 ) {
+ if ( deleteBatch.getRowCount() > 0 ) {
deleteBatch.execute();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index adbe03d..0ebb6a1 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -122,9 +122,9 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
try {
// loading will retrieve the oldest unique value entry for the field
- // purposely enable the read repair here to clean up before we write
- UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getReadCL(),
- written.getEntityId().getType(), Collections.singletonList(written.getField()), true);
+ // don't use read repair on this pre-write check
+ UniqueValueSet set = uniqueValueStrat.load(scope, written.getEntityId().getType(),
+ Collections.singletonList(written.getField()), false);
set.forEach(uniqueValue -> {
@@ -143,11 +143,15 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
throw new RuntimeException("Error connecting to cassandra", e);
}
- // use TTL in case something goes wrong before entity is finally committed
- final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
+ // only build the batch statement if we don't have a violation for the field
+ if( preWriteUniquenessViolations.get(field.getName()) != null) {
- batch.mergeShallow( mb );
- uniqueFields.add(field);
+ // use TTL in case something goes wrong before entity is finally committed
+ final MutationBatch mb = uniqueValueStrat.write(scope, written, serializationFig.getTimeout());
+
+ batch.mergeShallow(mb);
+ uniqueFields.add(field);
+ }
}
if(preWriteUniquenessViolations.size() > 0 ){
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
index 35bb1b8..c6c70b9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/UniqueValueSerializationStrategy.java
@@ -70,7 +70,23 @@ public interface UniqueValueSerializationStrategy extends Migration, VersionedDa
*
* @throws ConnectionException on error connecting to Cassandra
*/
- UniqueValueSet load( ApplicationScope applicationScope, String type, Collection<Field> fields ) throws ConnectionException;
+ UniqueValueSet load( ApplicationScope applicationScope, String type, Collection<Field> fields )
+ throws ConnectionException;
+
+ /**
+ * Load UniqueValue that matches field from collection or null if that value does not exist. Returns the oldest
+ * unique value entry if more than 1 exists
+ *
+ * @param applicationScope scope in which to look for field name/value
+ * @param type The type the unique value exists within
+ * @param fields Field name/value to search for
+ *
+ * @return UniqueValueSet containing fields from the collection that exist in cassandra
+ *
+ * @throws ConnectionException on error connecting to Cassandra
+ */
+ UniqueValueSet load( ApplicationScope applicationScope, String type, Collection<Field> fields,
+ boolean useReadRepair ) throws ConnectionException;
/**
* Load UniqueValue that matches field from collection or null if that value does not exist.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index db93272..4456123 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -240,6 +240,14 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
return load( colScope, ConsistencyLevel.valueOf( cassandraFig.getReadCL() ), type, fields, false);
}
+ @Override
+ public UniqueValueSet load( final ApplicationScope colScope, final String type, final Collection<Field> fields,
+ boolean useReadRepair)
+ throws ConnectionException {
+ return load( colScope, ConsistencyLevel.valueOf( cassandraFig.getReadCL() ), type, fields, useReadRepair);
+ }
+
+
@Override
public UniqueValueSet load(final ApplicationScope appScope, final ConsistencyLevel consistencyLevel,
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java
index b9c9999..f971b23 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyProxyImpl.java
@@ -113,6 +113,19 @@ public class UniqueValueSerializationStrategyProxyImpl implements UniqueValueSer
return migration.to.load( applicationScope, type, fields );
}
+ @Override
+ public UniqueValueSet load( final ApplicationScope applicationScope, final String type,
+ final Collection<Field> fields, boolean useReadRepair ) throws ConnectionException {
+
+ final MigrationRelationship<UniqueValueSerializationStrategy> migration = getMigrationRelationShip();
+
+ if ( migration.needsMigration() ) {
+ return migration.from.load( applicationScope, type, fields, useReadRepair );
+ }
+
+ return migration.to.load( applicationScope, type, fields, useReadRepair );
+ }
+
@Override
public UniqueValueSet load(final ApplicationScope applicationScope, final ConsistencyLevel consistencyLevel,
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index 1ed7255..8be86ab 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -82,7 +82,7 @@ public class AbstractCollectionService extends AbstractService {
nameProperty = "name";
}
- Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name, true);
+ Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name, false);
if ( entity != null ) {
entity = importEntity( request, entity );
}
@@ -172,7 +172,8 @@ public class AbstractCollectionService extends AbstractService {
public ServiceResults getItemByName( ServiceContext context, String name ) throws Exception {
// just get the UUID and then getItemById such that same results are being returned in both cases
- UUID entityId = em.getUniqueIdFromAlias( getEntityType(), name );
+ // don't use uniqueIndexRepair on read only logic
+ UUID entityId = em.getUniqueIdFromAlias( getEntityType(), name, false);
if ( entityId == null ) {
@@ -302,7 +303,7 @@ public class AbstractCollectionService extends AbstractService {
return getItemByName( context, name );
}
- // EntityRef ref = em.getAlias( getEntityType(), name );
+ // use unique index repair here before any write logic if there are problems
Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name, true);
if ( entity == null ) {
// null entity ref means we tried to put a non-existing entity
@@ -516,6 +517,7 @@ public class AbstractCollectionService extends AbstractService {
return super.postItemByName( context, name );
}
+ // use unique index repair here before any write logic if there are problems
Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name, true);
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
@@ -570,6 +572,7 @@ public class AbstractCollectionService extends AbstractService {
return getItemByName( context, name );
}
+ // use unique index repair here before any write logic if there are problems
Entity entity = em.getUniqueEntityFromAlias( getEntityType(), name, true);
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/82428785/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
index 5f7ca6b..3791644 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
@@ -271,7 +271,8 @@ public class AbstractConnectionsService extends AbstractService {
//TODO T.N. USERGRID-1919 actually validate this is connected
- Entity entity = em.getUniqueEntityFromAlias( query.getEntityType(), name, true);
+ // this is purely read only, don't use unique index repair
+ Entity entity = em.getUniqueEntityFromAlias( query.getEntityType(), name, false);
if ( entity == null ) {
return null;
}
@@ -372,6 +373,7 @@ public class AbstractConnectionsService extends AbstractService {
if ( query.containsSingleNameOrEmailIdentifier() ) {
String name = query.getSingleNameOrEmailIdentifier();
+ // use unique index repair here before any write logic if there are problems
entity = em.getUniqueEntityFromAlias( query.getEntityType(), name, true);
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
@@ -529,6 +531,7 @@ public class AbstractConnectionsService extends AbstractService {
nameProperty = "name";
}
+ // use unique index repair here before any write logic if there are problems
Entity entity = em.getUniqueEntityFromAlias( query.getEntityType(), name, true);
if ( entity == null ) {
throw new ServiceResourceNotFoundException( context );
[49/50] usergrid git commit: Fix getAllEdgeDocuments in deleteEdge to
return docs where entityId matches and not just the nodeId. Also fix test
that wasn't actually testing this.
Posted by mr...@apache.org.
Fix getAllEdgeDocuments in deleteEdge to return docs where entityId matches and not just the nodeId. Also fix test that wasn't actually testing this.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a8f4218a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a8f4218a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a8f4218a
Branch: refs/heads/master
Commit: a8f4218a7907db16297bd34cc859441b99a9c35c
Parents: 8faad5b
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Jul 12 17:09:43 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Jul 12 17:09:43 2016 -0700
----------------------------------------------------------------------
.../corepersistence/index/IndexServiceTest.java | 14 ++++++++++++--
.../persistence/index/impl/EsEntityIndexImpl.java | 8 ++++++--
2 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a8f4218a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
index fba6e7c..adecd9d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
@@ -355,6 +355,12 @@ public class IndexServiceTest {
final SearchEdge connectionSearchEdge = CpNamingUtils.createSearchEdgeFromSource( connectionSearch );
+ //ensure that no edges remain
+ CandidateResults connectionResultsEmpty = EntityIndex.search( connectionSearchEdge,
+ SearchTypes.fromTypes( "thing" ),"select *",10,0 );
+
+ assertEquals(1,connectionResultsEmpty.size());
+
//step 1
//(We need to mark then delete things in the graph manager.)
final Edge toBeMarkedEdge = graphManager.markEdge( connectionSearch ).toBlocking().firstOrDefault( null );
@@ -367,9 +373,13 @@ public class IndexServiceTest {
assertEquals( 1, indexOperationMessage.getDeIndexRequests().size() );
+ indexProducer.put(indexOperationMessage).toBlocking().last();
+
+ Thread.sleep(1000); // wait for the operation to flush at Elasticsearch
+
//ensure that no edges remain
- final CandidateResults connectionResultsEmpty = EntityIndex.search( connectionSearchEdge,
- SearchTypes.fromTypes( "things" ),"select *",10,0 );
+ connectionResultsEmpty = EntityIndex.search( connectionSearchEdge,
+ SearchTypes.fromTypes( "thing" ),"select *",10,0 );
assertEquals(0,connectionResultsEmpty.size());
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a8f4218a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index dc110f7..d2aff7e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -469,9 +469,12 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
// never let this fetch more than 100 to save memory
final int searchLimit = Math.min(100, indexFig.getVersionQueryLimit());
- final QueryBuilder entityQuery = QueryBuilders
+ final QueryBuilder nodeIdQuery = QueryBuilders
.termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(edge.getNodeId()));
+ final QueryBuilder entityIdQuery = QueryBuilders
+ .termQuery(IndexingUtils.ENTITY_ID_FIELDNAME, IndexingUtils.entityId(entityId));
+
final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder()
.addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC);
@@ -492,7 +495,8 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
QueryBuilder finalQuery = QueryBuilders
.boolQuery()
- .must(entityQuery)
+ .must(entityIdQuery)
+ .must(nodeIdQuery)
.must(timestampQuery);
searchResponse = srb
[06/50] usergrid git commit: Added app id to the other duplicate
unique value log.
Posted by mr...@apache.org.
Added app id to the other duplicate unique value log.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/09280b1f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/09280b1f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/09280b1f
Branch: refs/heads/master
Commit: 09280b1fea81938576e8143cdc4047b41aae4350
Parents: 0c965ae
Author: Mike Dunker <md...@apigee.com>
Authored: Mon Jun 27 17:04:27 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Mon Jun 27 17:04:27 2016 -0700
----------------------------------------------------------------------
.../impl/UniqueValueSerializationStrategyImpl.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/09280b1f/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index 28e28aa..aec2e58 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -361,9 +361,9 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
}else{
- logger.warn("Duplicate unique value [{}={}] found, removing newer entry " +
- "with entity id [{}] and entity version [{}]", field.getName(), field.getValue().toString(),
- uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion() );
+ logger.warn("Duplicate unique value [{}={}] found for application [{}], removing newer entry " +
+ "with entity id [{}] and entity version [{}].", field.getName(), field.getValue().toString(),
+ applicationId.getUuid(), uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion() );
// delete the duplicate from the unique value index
delete(appScope, uniqueValue ).execute();
[35/50] usergrid git commit: Revert "Fix tomcat dependency versions."
Posted by mr...@apache.org.
Revert "Fix tomcat dependency versions."
This reverts commit 3e65e78c33ca47404905cea8558d9e0e2800241f.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2cae8662
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2cae8662
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2cae8662
Branch: refs/heads/master
Commit: 2cae8662d13cc0e51380552a0e46a2f6b5163c27
Parents: 3e65e78
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jul 6 22:21:53 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jul 6 22:21:53 2016 -0700
----------------------------------------------------------------------
stack/pom.xml | 3 +--
stack/test-utils/pom.xml | 5 -----
2 files changed, 1 insertion(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2cae8662/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 5fd0817..1bf1f92 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -118,7 +118,6 @@
<slf4j-version>1.7.2</slf4j-version>
<snakeyaml-version>1.9</snakeyaml-version>
<tomcat-version>8.5.3</tomcat-version>
- <tomcat-logging-version>8.5.2</tomcat-logging-version>
<antlr.version>3.4</antlr.version>
<tika.version>1.4</tika.version>
<mockito.version>1.10.8</mockito.version>
@@ -909,7 +908,7 @@
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-logging-juli</artifactId>
- <version>${tomcat-logging-version}</version>
+ <version>${tomcat-version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2cae8662/stack/test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/stack/test-utils/pom.xml b/stack/test-utils/pom.xml
index b460198..c104f0a 100644
--- a/stack/test-utils/pom.xml
+++ b/stack/test-utils/pom.xml
@@ -245,31 +245,26 @@
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
- <version>${tomcat-version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-logging-juli</artifactId>
- <version>${tomcat-logging-version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-jasper</artifactId>
- <version>${tomcat-version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-jasper-el</artifactId>
- <version>${tomcat-version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-jsp-api</artifactId>
- <version>${tomcat-version}</version>
</dependency>
</dependencies>
[47/50] usergrid git commit: Merge commit 'refs/pull/543/head' of
github.com:apache/usergrid into release-2.1.1
Posted by mr...@apache.org.
Merge commit 'refs/pull/543/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4f8183ff
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4f8183ff
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4f8183ff
Branch: refs/heads/master
Commit: 4f8183ff1af70ca606f0ffb4ae7f7cde88b0647d
Parents: f95bf96 5f39ee0
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Jul 11 08:35:01 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Jul 11 08:35:01 2016 -0700
----------------------------------------------------------------------
docs/data-storage/collections.md | 144 ++++
.../main/resources/usergrid-default.properties | 44 +-
.../src/test/resources/usergrid-test.properties | 20 +-
stack/core/pom.xml | 817 ++++++++++---------
.../usergrid/corepersistence/CoreModule.java | 66 +-
.../corepersistence/CpEntityManager.java | 319 ++++----
.../corepersistence/CpEntityManagerFactory.java | 68 +-
.../corepersistence/CpRelationManager.java | 19 +-
.../corepersistence/EntityManagerFig.java | 9 +
.../index/CollectionSettingsCache.java | 47 ++
.../index/CollectionSettingsCacheFactory.java | 44 +
.../index/CollectionSettingsCacheFig.java | 39 +
.../index/CollectionSettingsCacheImpl.java | 122 +++
.../corepersistence/index/IndexSchemaCache.java | 50 --
.../index/IndexSchemaCacheFactory.java | 44 -
.../index/IndexSchemaCacheFig.java | 39 -
.../index/IndexSchemaCacheImpl.java | 120 ---
.../corepersistence/index/IndexServiceImpl.java | 54 +-
.../index/ReIndexServiceImpl.java | 44 +-
.../service/ApplicationServiceImpl.java | 22 +-
.../usergrid/persistence/EntityManager.java | 6 +-
.../index/AsyncIndexServiceTest.java | 2 +-
.../corepersistence/index/IndexServiceTest.java | 78 +-
.../resources/usergrid-custom-test.properties | 13 +
stack/corepersistence/actorsystem/pom.xml | 106 +++
.../persistence/actorsystem/ActorSystemFig.java | 83 ++
.../actorsystem/ActorSystemManager.java | 86 ++
.../actorsystem/ActorSystemManagerImpl.java | 459 +++++++++++
.../actorsystem/ActorSystemModule.java | 34 +
.../persistence/actorsystem/ClientActor.java | 205 +++++
.../actorsystem/GuiceActorProducer.java | 46 ++
.../persistence/actorsystem/RouterProducer.java | 51 ++
.../src/main/resources/application.conf | 50 ++
.../actorsystem/ActorServiceServiceTest.java | 74 ++
stack/corepersistence/collection/pom.xml | 120 ++-
.../collection/EntityCollectionManager.java | 8 +-
.../EntityCollectionManagerFactory.java | 4 +-
.../exception/WriteUniqueVerifyException.java | 2 +-
.../collection/guice/CollectionModule.java | 9 +
.../EntityCollectionManagerFactoryImpl.java | 167 ++--
.../impl/EntityCollectionManagerImpl.java | 141 ++--
.../mvcc/stage/CollectionIoEvent.java | 14 +-
.../mvcc/stage/delete/MarkCommit.java | 36 +-
.../mvcc/stage/write/WriteCommit.java | 82 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 105 ++-
.../impl/UniqueFieldRowKeySerializer.java | 3 +-
.../uniquevalues/ReservationCache.java | 87 ++
.../uniquevalues/ReservationCacheActor.java | 87 ++
.../uniquevalues/UniqueValueActor.java | 282 +++++++
.../uniquevalues/UniqueValueException.java | 33 +
.../uniquevalues/UniqueValuesFig.java | 67 ++
.../uniquevalues/UniqueValuesRouter.java | 70 ++
.../uniquevalues/UniqueValuesService.java | 74 ++
.../uniquevalues/UniqueValuesServiceImpl.java | 372 +++++++++
.../uniquevalues/UniqueValuesTable.java | 42 +
.../uniquevalues/UniqueValuesTableImpl.java | 94 +++
.../collection/AbstractUniqueValueTest.java | 50 ++
.../collection/EntityCollectionManagerIT.java | 171 ++--
.../EntityCollectionManagerStressTest.java | 21 +-
.../collection/guice/TestCollectionModule.java | 17 +
.../mvcc/stage/delete/MarkCommitTest.java | 10 +-
.../mvcc/stage/write/WriteCommitTest.java | 8 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 68 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 55 +-
.../UniqueValuesServiceDeleteTest.java | 149 ++++
.../uniquevalues/UniqueValuesServiceTest.java | 185 +++++
.../src/test/resources/usergrid-CHOP.properties | 18 +
.../src/test/resources/usergrid-UNIT.properties | 19 +
.../src/test/resources/usergrid.properties | 32 +
stack/corepersistence/common/pom.xml | 23 +-
.../core/guice/SettingsValidationCluster.java | 18 +
.../core/guice/MigrationManagerRule.java | 18 +
.../guice/SettingsValidationClusterTest.java | 19 +
.../src/test/resources/usergrid-UNIT.properties | 19 +
.../src/test/resources/usergrid.properties | 19 +
stack/corepersistence/model/pom.xml | 16 +
.../persistence/model/entity/EntityMap.java | 18 +
.../persistence/model/field/ArrayField.java | 7 +
.../persistence/model/field/ByteArrayField.java | 9 +
.../persistence/model/field/DistanceField.java | 9 +
.../model/field/EntityObjectField.java | 9 +
.../persistence/model/field/FieldTypeName.java | 3 -
.../persistence/model/field/ListField.java | 7 +
.../persistence/model/field/LocationField.java | 7 +
.../persistence/model/field/NullField.java | 7 +
.../persistence/model/field/SetField.java | 7 +
.../model/field/value/EntityObject.java | 2 +-
.../persistence/model/util/EntityUtils.java | 18 +
.../persistence/model/util/UUIDGenerator.java | 18 +
.../persistence/model/field/EntityTest.java | 18 +
.../model/util/UUIDGeneratorTest.java | 18 +
stack/corepersistence/pom.xml | 72 ++
stack/corepersistence/queryindex/pom.xml | 18 +-
stack/pom.xml | 6 -
.../usergrid/rest/AbstractContextResource.java | 1 +
.../apache/usergrid/rest/ShutdownListener.java | 11 +
.../rest/applications/CollectionResource.java | 97 +--
.../rest/applications/ServiceResource.java | 8 +-
...ApplicationAlreadyExistsExceptionMapper.java | 32 +
.../apache/usergrid/rest/NotificationsIT.java | 6 +-
.../org/apache/usergrid/rest/UniqueCatsIT.java | 233 ++++++
.../apache/usergrid/rest/UniqueValuesIT.java | 224 +++++
.../rest/UniqueValuesPerformanceIT.java | 165 ++++
.../rest/applications/ApplicationDeleteIT.java | 115 ++-
.../applications/ApplicationResourceIT.java | 2 +-
.../collection/CollectionsResourceIT.java | 251 +++---
.../resources/corepersistence-UNIT.properties | 2 +
.../resources/usergrid-custom-test.properties | 19 +-
.../resources/usergrid-rest-deploy-context.xml | 33 +-
stack/services/pom.xml | 6 +
.../services/AbstractCollectionService.java | 12 +-
.../services/AbstractConnectionsService.java | 4 +-
.../usergrid/services/AbstractService.java | 53 +-
.../applications/ApplicationsService.java | 4 +-
.../resources/usergrid-services-context.xml | 2 +-
115 files changed, 6158 insertions(+), 1633 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4f8183ff/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
[38/50] usergrid git commit: Fixes to tests and test properties.
Posted by mr...@apache.org.
Fixes to tests and test properties.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/917f0e32
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/917f0e32
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/917f0e32
Branch: refs/heads/master
Commit: 917f0e32ea72734b98902ef041a6fe5302495d4b
Parents: 996bf09
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 12:10:01 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 12:10:01 2016 -0400
----------------------------------------------------------------------
.../UniqueValuesServiceDeleteTest.java | 114 +++++++------------
1 file changed, 39 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/917f0e32/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
index 3f58834..397ec78 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
@@ -22,6 +22,7 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.inject.Inject;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.collection.AbstractUniqueValueTest;
@@ -52,6 +53,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import static junit.framework.TestCase.fail;
+
/**
* Test the unique values service.
@@ -86,7 +89,7 @@ public class UniqueValuesServiceDeleteTest extends AbstractUniqueValueTest {
@Before
public void initAkka() {
// each test class needs unique port number
- initAkka( 2555, actorSystemManager, uniqueValuesService );
+ initAkka( 2559, actorSystemManager, uniqueValuesService );
}
@@ -94,92 +97,53 @@ public class UniqueValuesServiceDeleteTest extends AbstractUniqueValueTest {
* Use multiple threads to attempt to create entities with duplicate usernames.
*/
@Test
- public void testDuplicatePrevention() throws Exception {
+ public void testUniqueValueCleanup() throws Exception {
initAkka();
- final AtomicInteger successCounter = new AtomicInteger( 0 );
- final AtomicInteger errorCounter = new AtomicInteger( 0 );
-
- Multimap<String, Entity> usersCreated =
- generateDuplicateUsers( numUsers, successCounter, errorCounter );
-
- int userCount = 0;
- int usernamesWithDuplicates = 0;
- for ( String username : usersCreated.keySet() ) {
- Collection<Entity> users = usersCreated.get( username );
- if ( users.size() > 1 ) {
- usernamesWithDuplicates++;
- }
- userCount++;
- }
-
- Assert.assertEquals( 0, usernamesWithDuplicates );
-
- Assert.assertEquals( numUsers, successCounter.get() );
- Assert.assertEquals( 0, errorCounter.get() );
- Assert.assertEquals( numUsers, usersCreated.size() );
- Assert.assertEquals( numUsers, userCount );
- }
-
-
- private Multimap<String, Entity> generateDuplicateUsers(
- int numUsers, AtomicInteger successCounter, AtomicInteger errorCounter ) {
-
ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
EntityCollectionManager manager = factory.createCollectionManager( context );
- Multimap<String, Entity> usersCreated =
- Multimaps.synchronizedListMultimap( ArrayListMultimap.create() );
-
- ExecutorService execService = Executors.newFixedThreadPool( poolSize );
+ String username = RandomStringUtils.randomAlphanumeric( 20 );
- for (int i = 0; i < numUsers; i++) {
-
- // multiple threads simultaneously trying to create a user with the same propertyName
- for (int j = 0; j < numThreads; j++) {
- String username = "user_" + i;
-
- execService.submit( () -> {
-
- try {
-
- // give entity two unqiue fields username and email
- Entity newEntity = new Entity( new SimpleId( "user" ) );
- newEntity.setField( new StringField( "username", username, true ) );
- newEntity.setField( new StringField( "email", username + "@example.org", true ) );
-
- Observable<Entity> observable = manager.write( newEntity, null );
- Entity returned = observable.toBlocking().lastOrDefault( null );
-
- usersCreated.put( username, newEntity );
- successCounter.incrementAndGet();
-
- logger.debug("Created user {}", username);
-
- } catch ( Throwable t ) {
- if ( t instanceof WriteUniqueVerifyException) {
- // we expect lots of these
- } else {
- errorCounter.incrementAndGet();
- logger.error( "Error creating user " + username, t );
- }
- }
+ // create user
+ Entity originalUser = null;
+ {
+ Entity newEntity = new Entity( new SimpleId( "user" ) );
+ newEntity.setField( new StringField( "username", username, true ) );
+ newEntity.setField( new StringField( "email", username + "@example.org", true ) );
+ Observable<Entity> observable = manager.write( newEntity, null );
+ originalUser = observable.toBlocking().lastOrDefault( null );
+ }
- } );
- }
+ // cannot create another user with same name
+ {
+ Entity newEntity = new Entity( new SimpleId( "user" ) );
+ newEntity.setField( new StringField( "username", username, true ) );
+ newEntity.setField( new StringField( "email", username + "@example.org", true ) );
+ try {
+ Observable<Entity> observable = manager.write( newEntity, null );
+ Entity returned = observable.toBlocking().lastOrDefault( null );
+ fail("Should not have created dupliate user");
+ } catch ( WriteUniqueVerifyException expected ) {}
}
- execService.shutdown();
- try {
- while (!execService.awaitTermination( 60, TimeUnit.SECONDS )) {
- System.out.println( "Waiting..." );
+ // delete user
+ manager.mark( originalUser.getId(), null ).toBlocking().firstOrDefault( null );
+
+ // now we can create another user with same name
+ {
+ Entity newEntity = new Entity( new SimpleId( "user" ) );
+ newEntity.setField( new StringField( "username", username, true ) );
+ newEntity.setField( new StringField( "email", username + "@example.org", true ) );
+ try {
+ Observable<Entity> observable = manager.write( newEntity, null );
+ Entity returned = observable.toBlocking().lastOrDefault( null );
+ } catch ( WriteUniqueVerifyException unexpected ) {
+ logger.error("Error creating user", unexpected);
+ fail("Still cannot create new user after delete");
}
- } catch (InterruptedException e) {
- e.printStackTrace();
}
-
- return usersCreated;
}
}
[43/50] usergrid git commit: Oops
Posted by mr...@apache.org.
Oops
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/47b76153
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/47b76153
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/47b76153
Branch: refs/heads/master
Commit: 47b761539c15e93c92fd5ac28806e2ff76039bea
Parents: 2fb55f0
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 15:23:41 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 15:23:41 2016 -0400
----------------------------------------------------------------------
stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/47b76153/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
index 6561389..0ccc18c 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import org.apache.commons.lang.RandomStringUtils;
import org.junit.Assert;
+import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
[32/50] usergrid git commit: Make the unique value loading use the
config interface for consistent read CL instead of hard coding to CL_ALL
Posted by mr...@apache.org.
Make the unique value loading use the config interface for consistent read CL instead of hard coding to CL_ALL
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bd96b181
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bd96b181
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bd96b181
Branch: refs/heads/master
Commit: bd96b18116e16fc35f2bc1d740cff65b03470594
Parents: 8982ddd
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jul 6 15:14:30 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jul 6 15:14:30 2016 -0700
----------------------------------------------------------------------
.../impl/EntityCollectionManagerFactoryImpl.java | 9 +++++++--
.../collection/impl/EntityCollectionManagerImpl.java | 11 ++++++++---
.../collection/mvcc/stage/write/WriteUniqueVerify.java | 4 ++--
3 files changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bd96b181/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index a52ee9c..01eab7d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@ -40,6 +40,7 @@ import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerial
import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -77,6 +78,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
private final Keyspace keyspace;
private final MetricsFactory metricsFactory;
private final RxTaskScheduler rxTaskScheduler;
+ private final CassandraConfig cassandraConfig;
private LoadingCache<ApplicationScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
@@ -89,7 +91,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
entitySerializationStrategy, uniqueValueSerializationStrategy,
mvccLogEntrySerializationStrategy, keyspace,
metricsFactory, serializationFig,
- rxTaskScheduler, scope );
+ rxTaskScheduler, scope, cassandraConfig );
return target;
}
@@ -107,7 +109,9 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace, final EntityCacheFig entityCacheFig,
- final MetricsFactory metricsFactory, @CollectionExecutorScheduler final RxTaskScheduler rxTaskScheduler ) {
+ final MetricsFactory metricsFactory,
+ @CollectionExecutorScheduler final RxTaskScheduler rxTaskScheduler,
+ final CassandraConfig cassandraConfig) {
this.writeStart = writeStart;
this.writeVerifyUnique = writeVerifyUnique;
@@ -125,6 +129,7 @@ public class EntityCollectionManagerFactoryImpl implements EntityCollectionManag
this.keyspace = keyspace;
this.metricsFactory = metricsFactory;
this.rxTaskScheduler = rxTaskScheduler;
+ this.cassandraConfig = cassandraConfig;
}
@Override
public EntityCollectionManager createCollectionManager(ApplicationScope applicationScope) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bd96b181/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 9261b56..06b086c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -28,6 +28,7 @@ import java.util.UUID;
import com.netflix.astyanax.model.ConsistencyLevel;
import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator;
+import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -110,6 +111,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
private final SerializationFig serializationFig;
+ private final CassandraConfig cassandraConfig;
private final Keyspace keyspace;
@@ -135,7 +137,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
final Keyspace keyspace, final MetricsFactory metricsFactory,
final SerializationFig serializationFig, final RxTaskScheduler rxTaskScheduler,
- @Assisted final ApplicationScope applicationScope ) {
+ @Assisted final ApplicationScope applicationScope,
+ final CassandraConfig cassandraConfig) {
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
this.uniqueCleanup = uniqueCleanup;
@@ -166,6 +169,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.fieldEntityTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "base.fieldEntity");
this.loadTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "base.load");
this.getLatestTimer = metricsFactory.getTimer(EntityCollectionManagerImpl.class, "base.latest");
+
+ this.cassandraConfig = cassandraConfig;
}
@@ -333,10 +338,10 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final UUID startTime = UUIDGenerator.newTimeUUID();
//Get back set of unique values that correspond to collection of fields
- //Purposely use CL ALL as consistency is extremely important here, regardless of performance
+ //Purposely use string consistency as it's extremely important here, regardless of performance
UniqueValueSet set =
uniqueValueSerializationStrategy
- .load( applicationScope, ConsistencyLevel.CL_ALL, type, fields1 , uniqueIndexRepair);
+ .load( applicationScope, cassandraConfig.getConsistentReadCL(), type, fields1 , uniqueIndexRepair);
//Short circuit if we don't have any uniqueValues from the given fields.
if ( !set.iterator().hasNext() ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bd96b181/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 01e6621..b36cb79 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -123,8 +123,8 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
// loading will retrieve the oldest unique value entry for the field
// don't use read repair on this pre-write check
- // use CL ALL as consistency is extremely important here, more so than performance
- UniqueValueSet set = uniqueValueStrat.load(scope, ConsistencyLevel.CL_ALL,
+ // stronger consistency is extremely important here, more so than performance
+ UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getConsistentReadCL(),
written.getEntityId().getType(), Collections.singletonList(written.getField()), false);
[24/50] usergrid git commit: Fix port settings for tests
Posted by mr...@apache.org.
Fix port settings for tests
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5ddb1cd7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5ddb1cd7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5ddb1cd7
Branch: refs/heads/master
Commit: 5ddb1cd7348b075fa16b66683cb626e11b4d0861
Parents: 1b4e117
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jul 1 15:17:27 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jul 1 15:17:27 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 ++
.../actorsystem/ActorSystemManager.java | 7 ++++
.../actorsystem/ActorSystemManagerImpl.java | 6 +++
.../uniquevalues/UniqueValuesServiceImpl.java | 22 ++++++++---
.../collection/AbstractUniqueValueTest.java | 2 +-
.../collection/EntityCollectionManagerIT.java | 2 +-
.../EntityCollectionManagerStressTest.java | 19 +++++++++-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +-
.../uniquevalues/UniqueValuesServiceTest.java | 2 +-
.../src/test/resources/usergrid.properties | 39 +++++++++++++-------
11 files changed, 81 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index fe70569..95b3e9b 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -431,6 +431,9 @@ usergrid.cluster.seeds=default:localhost
# Port used for cluster communications.
usergrid.cluster.port=2551
+# If unspecified then system will use InetAddress.getLocalHost().getHostName()
+usergrid.cluster.hostname=
+
######################### Usergrid Unique Values Validation ##################
#
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
index cdb6caf..893afca 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
@@ -21,6 +21,8 @@ package org.apache.usergrid.persistence.actorsystem;
import akka.actor.ActorRef;
+import java.util.Set;
+
public interface ActorSystemManager {
/**
@@ -71,6 +73,11 @@ public interface ActorSystemManager {
String getCurrentRegion();
/**
+ * Get all regions known to system.
+ */
+ public Set<String> getRegions();
+
+ /**
* Publish message to all topic subscribers in all regions.
*/
void publishToAllRegions( String topic, Object message, ActorRef sender );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index a79f447..099b875 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -67,12 +67,18 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
private ListMultimap<String, String> seedsByRegion;
+
@Inject
public ActorSystemManagerImpl( ActorSystemFig actorSystemFig ) {
this.actorSystemFig = actorSystemFig;
}
+ public Set<String> getRegions() {
+ return getSeedsByRegion().keySet();
+ }
+
+
/**
* Init Akka ActorSystems and wait for request actors to start.
*/
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
index 352c2e5..fcee8fa 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
@@ -32,6 +32,7 @@ import akka.util.Timeout;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Singleton;
+import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.actorsystem.GuiceActorProducer;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -92,9 +93,7 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
public void reserveUniqueValues(
ApplicationScope scope, Entity entity, UUID version, String region ) throws UniqueValueException {
- if ( !actorSystemManager.isReady() ) {
- throw new RuntimeException("Unique values service not initialized, no request actors ready");
- }
+ ready();
try {
for (Field field : entity.getFields()) {
@@ -124,9 +123,7 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
public void confirmUniqueValues(
ApplicationScope scope, Entity entity, UUID version, String region ) throws UniqueValueException {
- if ( !actorSystemManager.isReady() ) {
- throw new RuntimeException("Unique values service not initialized, no request actors ready");
- }
+ ready();
try {
for (Field field : entity.getFields()) {
@@ -202,6 +199,19 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
}
+ private void ready() {
+ if ( !actorSystemManager.isReady() ) {
+ throw new RuntimeException("Unique values service not initialized, no request actors ready");
+ }
+
+ if ( !StringUtils.isEmpty( uniqueValuesFig.getAuthoritativeRegion() )) {
+ if ( !actorSystemManager.getRegions().contains( uniqueValuesFig.getAuthoritativeRegion() ) ) {
+ throw new RuntimeException( "Authoritative region not in region list" );
+ }
+ }
+ }
+
+
private void sendUniqueValueRequest(
Entity entity, String region, UniqueValueActor.Request request ) throws UniqueValueException {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
index e7b4450..3bfc48b 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
@@ -40,7 +40,7 @@ public class AbstractUniqueValueTest {
actorSystemManager.registerMessageType( UniqueValueActor.Reservation.class, "/user/uvProxy" );
actorSystemManager.registerMessageType( UniqueValueActor.Cancellation.class, "/user/uvProxy" );
actorSystemManager.registerMessageType( UniqueValueActor.Confirmation.class, "/user/uvProxy" );
- actorSystemManager.start( "127.0.0.1", port, "us-east" );
+ actorSystemManager.start( "localhost", port, "us-east" );
actorSystemManager.waitForClientActor();
startedAkka.put( port, true );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index f8322c6..ceda1a8 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -89,7 +89,7 @@ public class EntityCollectionManagerIT extends AbstractUniqueValueTest {
@Before
public void initAkka() {
- initAkka( 2555, actorSystemManager, uniqueValuesService );
+ initAkka( 2551, actorSystemManager, uniqueValuesService );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
index 7904a42..562a5fc 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
@@ -22,6 +22,9 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.usergrid.StressTest;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -52,7 +55,7 @@ import static org.junit.Assert.assertNotNull;
@RunWith(ITRunner.class)
@UseModules(TestCollectionModule.class)
@Category(StressTest.class)
-public class EntityCollectionManagerStressTest {
+public class EntityCollectionManagerStressTest extends AbstractUniqueValueTest {
private static final Logger logger = LoggerFactory.getLogger(
EntityCollectionManagerStressTest.class );
@@ -63,6 +66,20 @@ public class EntityCollectionManagerStressTest {
@Rule
public MigrationManagerRule migrationManagerRule;
+ @Inject
+ ActorSystemManager actorSystemManager;
+
+ @Inject
+ UniqueValuesService uniqueValuesService;
+
+
+ @Before
+ public void initAkka() {
+ // each test class needs unique port number
+ initAkka( 2552, actorSystemManager, uniqueValuesService );
+ }
+
+
@Test
public void writeThousands() {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
index 703d6c2..c70cd1a 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
@@ -85,7 +85,7 @@ public class WriteUniqueVerifyIT extends AbstractUniqueValueTest {
@Before
public void initAkka() {
// each test class needs unique port number
- initAkka( 2551, actorSystemManager, uniqueValuesService );
+ initAkka( 2553, actorSystemManager, uniqueValuesService );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
index 46cfde1..7afba05 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
@@ -77,7 +77,7 @@ public class WriteUniqueVerifyTest extends AbstractUniqueValueTest {
@Before
public void initAkka() {
// each test class needs unique port number
- initAkka( 2552, actorSystemManager, uniqueValuesService );
+ initAkka( 2554, actorSystemManager, uniqueValuesService );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
index 672e3bf..1788a88 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
@@ -86,7 +86,7 @@ public class UniqueValuesServiceTest extends AbstractUniqueValueTest {
@Before
public void initAkka() {
// each test class needs unique port number
- initAkka( 2553, actorSystemManager, uniqueValuesService );
+ initAkka( 2555, actorSystemManager, uniqueValuesService );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid.properties b/stack/corepersistence/collection/src/test/resources/usergrid.properties
index 759a3b3..8c999e2 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid.properties
@@ -1,17 +1,30 @@
-# This property is required to be set and cannot be defaulted anywhere
-usergrid.cluster_name=usergrid
-collection.akka.enabled=true
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
-collection.akka.hostname=localhost
-collection.akka.port=2551
-collection.akka.region=us-east
-usergrid.queue.regionList=us-east
-collection.akka.authoritative.region=us-east
-collection.akka.region.seeds=us-east\:localhost\:2551
+# This property is required to be set and cannot be defaulted anywhere
+usergrid.cluster_name=usergrid
-collection.akka.uniquevalue.actors=300
-collection.akka.uniquevalue.cache.ttl=10
-collection.akka.uniquevalue.reservation.ttl=10
+usergrid.cluster.hostname=localhost
+usergrid.cluster.enabled=true
+usergrid.cluster.region.local=us-east
+usergrid.cluster.region.list=us-east
+usergrid.cluster.seeds=us-east\:localhost
-collection.akka.uniquevalue.region.types
+collection.uniquevalues.actors=300
+collection.uniquevalues.authoritative.region=us-east
[46/50] usergrid git commit: Fix REST test issues and implement a way
to shutdown the Akka actor system, with hooks for JVM stop/servlet stop.
Posted by mr...@apache.org.
Fix REST test issues and implement a way to shutdown the Akka actor system, with hooks for JVM stop/servlet stop.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5f39ee0a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5f39ee0a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5f39ee0a
Branch: refs/heads/master
Commit: 5f39ee0af119305ef1a63c7fb5b19e08db8a7890
Parents: 31b2040
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jul 8 15:33:26 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jul 8 15:33:26 2016 -0700
----------------------------------------------------------------------
.../src/test/resources/usergrid-test.properties | 5 +-
.../actorsystem/ActorSystemManager.java | 2 +
.../actorsystem/ActorSystemManagerImpl.java | 81 +++++++++++++-------
.../uniquevalues/ReservationCacheActor.java | 25 ++++--
.../uniquevalues/UniqueValueActor.java | 4 +-
.../apache/usergrid/rest/ShutdownListener.java | 11 +++
.../applications/ApplicationResourceIT.java | 2 +-
.../resources/usergrid-custom-test.properties | 6 +-
.../resources/usergrid-rest-deploy-context.xml | 1 -
9 files changed, 96 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f39ee0a/stack/config/src/test/resources/usergrid-test.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/test/resources/usergrid-test.properties b/stack/config/src/test/resources/usergrid-test.properties
index 368c585..085a11d 100644
--- a/stack/config/src/test/resources/usergrid-test.properties
+++ b/stack/config/src/test/resources/usergrid-test.properties
@@ -30,6 +30,9 @@
# This property is required to be set and cannot be defaulted anywhere
usergrid.cluster_name=usergrid
+# Set this for testing purposesly only
+usergrid.test=true
+
# Whether to user the remote Cassandra cluster or not
cassandra.use_remote=false
@@ -153,7 +156,7 @@ groupid=counter_group
autooffset.reset=smallest
# set high batch size to minimize count overhead
-usergrid.counter.batch.size=10000
+usergrid.counter.batch.size=1
usergrid.recaptcha.public=
usergrid.recaptcha.private=
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f39ee0a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
index 893afca..c7322dd 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
@@ -81,4 +81,6 @@ public interface ActorSystemManager {
* Publish message to all topic subscribers in all regions.
*/
void publishToAllRegions( String topic, Object message, ActorRef sender );
+
+ void shutdownAll();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f39ee0a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index 8399979..d8d284f 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -66,6 +66,8 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
private ListMultimap<String, String> seedsByRegion;
+ private ActorSystem clusterSystem = null;
+
@Inject
@@ -173,15 +175,15 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
// Create one actor system with request actor for each region
if ( StringUtils.isEmpty( currentRegion )) {
- throw new RuntimeException( "No value specified for " + ActorSystemFig.CLUSTER_REGIONS_LOCAL );
+ throw new RuntimeException( "No value specified for: " + ActorSystemFig.CLUSTER_REGIONS_LOCAL );
}
if ( StringUtils.isEmpty( actorSystemFig.getRegionsList() )) {
- throw new RuntimeException( "No value specified for " + ActorSystemFig.CLUSTER_REGIONS_LIST );
+ throw new RuntimeException( "No value specified for: " + ActorSystemFig.CLUSTER_REGIONS_LIST );
}
if ( StringUtils.isEmpty( actorSystemFig.getSeeds() )) {
- throw new RuntimeException( "No value specified for " + ActorSystemFig.CLUSTER_SEEDS );
+ throw new RuntimeException( "No value specified for: " + ActorSystemFig.CLUSTER_SEEDS );
}
List regionList = Arrays.asList( actorSystemFig.getRegionsList().toLowerCase().split(",") );
@@ -191,15 +193,15 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
Config config = readClusterSystemConfig();
- ActorSystem localSystem = createClusterSystemsFromConfigs( config );
+ clusterSystem = createClusterSystemsFromConfigs( config );
- createClientActors( localSystem );
+ createClientActors( clusterSystem );
for ( RouterProducer routerProducer : routerProducers ) {
- routerProducer.createLocalSystemActors( localSystem );
+ routerProducer.createLocalSystemActors( clusterSystem );
}
- mediator = DistributedPubSub.get( localSystem ).mediator();
+ mediator = DistributedPubSub.get( clusterSystem ).mediator();
}
@@ -214,7 +216,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
String[] regionSeeds = actorSystemFig.getSeeds().split( "," );
- logger.info( "Found region {} seeds {}", regionSeeds.length, regionSeeds );
+ logger.info( "Found region [{}] seeds [{}]", regionSeeds.length, regionSeeds );
try {
@@ -248,7 +250,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
String seed = "akka.tcp://ClusterSystem" + "@" + hostname + ":" + regionPort;
- logger.info( "Adding seed {} for region {}", seed, region );
+ logger.info( "Adding seed [{}] for region [{}]", seed, region );
seedsByRegion.put( region, seed );
}
@@ -283,7 +285,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
List<String> seeds = getSeedsByRegion().get( region );
- logger.info( "Akka Config for region {} is:\n" + " Hostname {}\n" + " Seeds {}\n",
+ logger.info( "Akka Config for region [{}] is:\n" + " Hostname [{}]\n" + " Seeds [{}]\n",
region, hostname, seeds );
int lastColon = seeds.get(0).lastIndexOf(":") + 1;
@@ -335,19 +337,38 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
*/
private ActorSystem createClusterSystemsFromConfigs( Config config ) {
- ActorSystem system = ActorSystem.create( "ClusterSystem", config );
- for ( RouterProducer routerProducer : routerProducers ) {
- logger.info("Creating {} for region {}", routerProducer.getName(), currentRegion );
- routerProducer.createClusterSingletonManager( system );
- }
+ // there is only 1 akka system for a Usergrid cluster
+ final String clusterName = "ClusterSystem";
+
+
+ if( clusterSystem == null) {
+
+ logger.info("Class: {}. ActorSystem [{}] not initialized, creating...", this, clusterName);
+
+ clusterSystem = ActorSystem.create( clusterName, config );
+
+ for ( RouterProducer routerProducer : routerProducers ) {
+ logger.info("Creating router producer [{}] for region [{}]", routerProducer.getName(), currentRegion );
+ routerProducer.createClusterSingletonManager( clusterSystem );
+ }
+
+ for ( RouterProducer routerProducer : routerProducers ) {
+ logger.info("Creating [{}] proxy for region [{}] role 'io'", routerProducer.getName(), currentRegion);
+ routerProducer.createClusterSingletonProxy( clusterSystem, "io" );
+ }
+
+ //add a shutdown hook to clean all actor systems if the JVM exits without the servlet container knowing
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ shutdownAll();
+ }
+ });
- for ( RouterProducer routerProducer : routerProducers ) {
- logger.info("Creating {} proxy for region {} role 'io'", routerProducer.getName(), currentRegion);
- routerProducer.createClusterSingletonProxy( system, "io" );
}
- return system;
+ return clusterSystem;
}
@@ -360,7 +381,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
if ( currentRegion.equals( region )) {
- logger.info( "Creating clientActor for region {}", region );
+ logger.info( "Creating clientActor for region [{}]", region );
// Each clientActor needs to know path to ClusterSingletonProxy and region
clientActor = system.actorOf(
@@ -381,7 +402,6 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
clusterClientsByRegion.put( region, clusterClient );
}
-
}
}
@@ -394,7 +414,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
private void waitForClientActor( ActorRef ra ) {
- logger.info( "Waiting on request actor {}...", ra.path() );
+ logger.info( "Waiting on RequestActor [{}]...", ra.path() );
started = false;
@@ -411,20 +431,29 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
started = true;
break;
}
- logger.info( "Waiting for request actor {} region {} ({}s)", ra.path(), currentRegion, retries );
+ logger.info( "Waiting for RequestActor [{}] region [{}] for [{}s]", ra.path(), currentRegion, retries );
Thread.sleep( 1000 );
} catch (Exception e) {
- logger.error( "Error: Timeout waiting for requestActor" );
+ logger.error( "Error: Timeout waiting for RequestActor [{}]", ra.path() );
}
retries++;
}
if (started) {
- logger.info( "RequestActor has started" );
+ logger.info( "RequestActor [{}] has started", ra.path() );
} else {
- throw new RuntimeException( "RequestActor did not start in time" );
+ throw new RuntimeException( "RequestActor ["+ra.path()+"] did not start in time" );
}
}
+ @Override
+ public void shutdownAll(){
+
+ logger.info("Shutting down Akka cluster: {}", clusterSystem.name());
+ clusterSystem.shutdown();
+
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f39ee0a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
index 158b099..3998eb6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
@@ -49,27 +49,36 @@ public class ReservationCacheActor extends UntypedActor {
ReservationCache.getInstance().cacheReservation( res );
if ( ++reservationCount % 10 == 0 ) {
- logger.info("Received {} reservations cache size {}",
+ if(logger.isDebugEnabled()) {
+ logger.debug("Received {} reservations cache size {}",
reservationCount, ReservationCache.getInstance().getSize());
+ }
}
} else if ( msg instanceof UniqueValueActor.Cancellation ) {
UniqueValueActor.Cancellation can = (UniqueValueActor.Cancellation) msg;
- ReservationCache.getInstance().cancelReservation( can );
-
- if (++cancellationCount % 10 == 0) {
- logger.info( "Received {} cancellations", cancellationCount );
+ ReservationCache.getInstance().cancelReservation(can);
+ if (logger.isDebugEnabled()) {
+ if (++cancellationCount % 10 == 0) {
+ logger.debug("Received {} cancellations", cancellationCount);
+ } else {
+ logger.debug("Removing cancelled {} from reservation cache", can.getConsistentHashKey());
+ }
}
- logger.debug("Removing cancelled {} from reservation cache", can.getConsistentHashKey());
+
} else if ( msg instanceof UniqueValueActor.Response ) {
UniqueValueActor.Response response = (UniqueValueActor.Response) msg;
ReservationCache.getInstance().cancelReservation( response );
- logger.info("Removing completed {} from reservation cache", response.getConsistentHashKey());
+ if(logger.isDebugEnabled()) {
+ logger.debug("Removing completed {} from reservation cache", response.getConsistentHashKey());
+ }
} else if (msg instanceof DistributedPubSubMediator.SubscribeAck) {
- logger.debug( "subscribing" );
+ if(logger.isDebugEnabled()) {
+ logger.debug("subscribing");
+ }
} else {
unhandled( msg );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f39ee0a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
index 501037f..74f45eb 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
@@ -57,8 +57,8 @@ public class UniqueValueActor extends UntypedActor {
Request req = (Request) message;
count++;
- if (count % 10 == 0) {
- logger.info( "UniqueValueActor {} processed {} requests", name, count );
+ if (count % 10 == 0 && logger.isDebugEnabled()) {
+ logger.debug( "UniqueValueActor {} processed {} requests", name, count );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f39ee0a/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java b/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java
index f9f5421..f9f1653 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/ShutdownListener.java
@@ -18,8 +18,10 @@
package org.apache.usergrid.rest;
+import com.google.inject.Injector;
import org.apache.usergrid.batch.service.JobSchedulerService;
import org.apache.usergrid.batch.service.SchedulerService;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,6 +65,15 @@ public class ShutdownListener implements ServletContextListener {
logger.info("ShutdownListener invoked");
+ ApplicationContext ctx = WebApplicationContextUtils
+ .getWebApplicationContext(sce.getServletContext());
+
+ Injector injector = ctx.getBean(Injector.class);
+ ActorSystemManager actorSystemManager = injector.getInstance(ActorSystemManager.class);
+
+ // stop the Akka actor system
+ actorSystemManager.shutdownAll();
+
boolean started = Boolean.parseBoolean(
properties.getProperty(JobServiceBoostrap.START_SCHEDULER_PROP, "true"));
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f39ee0a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
index 06615df..9f4f8aa 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
@@ -279,7 +279,7 @@ public class ApplicationResourceIT extends AbstractRestIT {
.get(ApiResponse.class);
// assert that the response returns the correct URI
- assertEquals(apiResponse.getUri(), String.format("http://sometestvalue/%s/%s", orgName, appName));
+ assertEquals(String.format("http://localhost:8080/%s/%s", orgName, appName), apiResponse.getUri());
//unmarshal the application from the response
Application application = new Application(apiResponse);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f39ee0a/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index f20f1e5..d845fcc 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -35,8 +35,6 @@ collection.stage.transient.timeout=5
# other...
usergrid.mongo.disable=true
-usergrid.counter.batch.size=1
-usergrid.api.url.base=http://sometestvalue
usergrid.notifications.listener.run=false
@@ -63,5 +61,9 @@ usergrid.cluster.region.local=us-east
usergrid.cluster.region.list=us-east
usergrid.cluster.seeds=us-east\:localhost
+# Use random port here for REST tests run outside embedded tomcat because these will get an instance of Spring that
+# starts the Akka cluster, then the embedded tomcat will also try when it starts ( but using default props and port 2551)
+usergrid.cluster.port=2555
+
collection.uniquevalues.actors=300
collection.uniquevalues.authoritative.region=us-east
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f39ee0a/stack/rest/src/test/resources/usergrid-rest-deploy-context.xml
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-rest-deploy-context.xml b/stack/rest/src/test/resources/usergrid-rest-deploy-context.xml
index 9cc5ea6..07215ab 100644
--- a/stack/rest/src/test/resources/usergrid-rest-deploy-context.xml
+++ b/stack/rest/src/test/resources/usergrid-rest-deploy-context.xml
@@ -29,7 +29,6 @@
<list>
<value>classpath:/usergrid-default.properties</value>
<value>classpath:/usergrid-test.properties</value>
- <value>classpath:/usergrid-custom-test-rest.properties</value>
</list>
</property>
</bean>
[40/50] usergrid git commit: Merge branch 'release-2.1.1' of
https://git-wip-us.apache.org/repos/asf/usergrid into usergrid-1268-akka-211
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into usergrid-1268-akka-211
# Conflicts:
# stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
# stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/37972a2e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/37972a2e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/37972a2e
Branch: refs/heads/master
Commit: 37972a2ed8c21c144f7e56d76afef37775f5012d
Parents: e6c59ee 86ae2cd
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 12:30:10 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 12:30:10 2016 -0400
----------------------------------------------------------------------
.../EntityCollectionManagerFactoryImpl.java | 103 ++++++++---------
.../impl/EntityCollectionManagerImpl.java | 112 ++++++++-----------
.../mvcc/stage/write/WriteUniqueVerify.java | 4 +-
3 files changed, 103 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/37972a2e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
index 3877fe3,01eab7d..fcaa51d
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerFactoryImpl.java
@@@ -19,38 -21,37 +19,33 @@@
package org.apache.usergrid.persistence.collection.impl;
--import java.util.concurrent.ExecutionException;
--
++import com.google.common.base.Preconditions;
++import com.google.common.cache.CacheBuilder;
++import com.google.common.cache.CacheLoader;
++import com.google.common.cache.LoadingCache;
++import com.google.inject.Inject;
++import com.google.inject.Singleton;
++import com.netflix.astyanax.Keyspace;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
--import org.apache.usergrid.persistence.collection.cache.EntityCacheFig;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.UniqueCleanup;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.VersionCompact;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
++import org.apache.usergrid.persistence.collection.mvcc.stage.write.*;
import org.apache.usergrid.persistence.collection.scheduler.CollectionExecutorScheduler;
import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
+ import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
--import com.google.common.base.Preconditions;
--import com.google.common.cache.CacheBuilder;
--import com.google.common.cache.CacheLoader;
--import com.google.common.cache.LoadingCache;
--import com.google.inject.Inject;
--import com.google.inject.Singleton;
--import com.netflix.astyanax.Keyspace;
++import java.util.concurrent.ExecutionException;
@@@ -60,7 -61,7 +55,6 @@@
@Singleton
public class EntityCollectionManagerFactoryImpl implements EntityCollectionManagerFactory {
--
private final WriteStart writeStart;
private final WriteUniqueVerify writeVerifyUnique;
private final WriteOptimisticVerify writeOptimisticVerify;
@@@ -77,35 -78,20 +71,40 @@@
private final Keyspace keyspace;
private final MetricsFactory metricsFactory;
private final RxTaskScheduler rxTaskScheduler;
+ private final ActorSystemManager actorSystemManager;
+ private final UniqueValuesService uniqueValuesService;
+
+ private final CassandraConfig cassandraConfig;
private LoadingCache<ApplicationScope, EntityCollectionManager> ecmCache =
CacheBuilder.newBuilder().maximumSize( 1000 )
.build( new CacheLoader<ApplicationScope, EntityCollectionManager>() {
public EntityCollectionManager load( ApplicationScope scope ) {
- //create the target EM that will perform logic
+ //create the target EM that will perform logic
final EntityCollectionManager target = new EntityCollectionManagerImpl(
- writeStart, writeVerifyUnique,
- writeOptimisticVerify, writeCommit, rollback, markStart, markCommit, uniqueCleanup, versionCompact,
- entitySerializationStrategy, uniqueValueSerializationStrategy,
- mvccLogEntrySerializationStrategy, keyspace,
- metricsFactory, serializationFig,
- rxTaskScheduler, scope, cassandraConfig );
++
+ writeStart,
+ writeVerifyUnique,
+ writeOptimisticVerify,
+ writeCommit,
+ rollback,
+ markStart,
+ markCommit,
+ uniqueCleanup,
+ versionCompact,
++
+ entitySerializationStrategy,
+ uniqueValueSerializationStrategy,
+ mvccLogEntrySerializationStrategy,
++
+ keyspace,
+ metricsFactory,
+ serializationFig,
+ rxTaskScheduler,
+ actorSystemManager,
+ uniqueValuesService,
++ cassandraConfig,
+ scope );
return target;
}
@@@ -113,47 -99,38 +112,49 @@@
@Inject
- public EntityCollectionManagerFactoryImpl( final WriteStart writeStart, final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit, final RollbackAction rollback,
- final MarkStart markStart, final MarkCommit markCommit,
- final UniqueCleanup uniqueCleanup, final VersionCompact versionCompact,
- final SerializationFig serializationFig, final
- MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace, final EntityCacheFig entityCacheFig,
- final MetricsFactory metricsFactory,
- @CollectionExecutorScheduler final RxTaskScheduler rxTaskScheduler,
- final CassandraConfig cassandraConfig) {
-
- this.writeStart = writeStart;
- this.writeVerifyUnique = writeVerifyUnique;
- this.writeOptimisticVerify = writeOptimisticVerify;
- this.writeCommit = writeCommit;
- this.rollback = rollback;
- this.markStart = markStart;
- this.markCommit = markCommit;
- this.uniqueCleanup = uniqueCleanup;
- this.versionCompact = versionCompact;
- this.serializationFig = serializationFig;
- this.entitySerializationStrategy = entitySerializationStrategy;
- this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
+ public EntityCollectionManagerFactoryImpl(
- final WriteStart writeStart,
- final WriteUniqueVerify writeVerifyUnique,
++ final WriteStart writeStart,
++ final WriteUniqueVerify writeVerifyUnique,
+ final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit,
- final RollbackAction rollback,
- final MarkStart markStart,
- final MarkCommit markCommit,
- final UniqueCleanup uniqueCleanup,
- final VersionCompact versionCompact,
- final SerializationFig serializationFig,
- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
++ final WriteCommit writeCommit,
++ final RollbackAction rollback,
++ final MarkStart markStart,
++ final MarkCommit markCommit,
++ final UniqueCleanup uniqueCleanup,
++ final VersionCompact versionCompact,
++ final SerializationFig serializationFig,
++ final MvccEntitySerializationStrategy entitySerializationStrategy,
++ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace,
- final EntityCacheFig entityCacheFig,
- final MetricsFactory metricsFactory, @CollectionExecutorScheduler
- final RxTaskScheduler rxTaskScheduler,
- final ActorSystemManager actorSystemManager,
- final UniqueValuesService uniqueValuesService ) {
-
- this.writeStart = writeStart;
- this.writeVerifyUnique = writeVerifyUnique;
- this.writeOptimisticVerify = writeOptimisticVerify;
- this.writeCommit = writeCommit;
- this.rollback = rollback;
- this.markStart = markStart;
- this.markCommit = markCommit;
- this.uniqueCleanup = uniqueCleanup;
- this.versionCompact = versionCompact;
- this.serializationFig = serializationFig;
- this.entitySerializationStrategy = entitySerializationStrategy;
- this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
++ final Keyspace keyspace,
++ final MetricsFactory metricsFactory,
++ @CollectionExecutorScheduler
++ final RxTaskScheduler rxTaskScheduler,
++ final ActorSystemManager actorSystemManager,
++ final UniqueValuesService uniqueValuesService,
++ final CassandraConfig cassandraConfig ) {
++
++ this.writeStart = writeStart;
++ this.writeVerifyUnique = writeVerifyUnique;
++ this.writeOptimisticVerify = writeOptimisticVerify;
++ this.writeCommit = writeCommit;
++ this.rollback = rollback;
++ this.markStart = markStart;
++ this.markCommit = markCommit;
++ this.uniqueCleanup = uniqueCleanup;
++ this.versionCompact = versionCompact;
++ this.serializationFig = serializationFig;
++ this.entitySerializationStrategy = entitySerializationStrategy;
++ this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
-- this.keyspace = keyspace;
-- this.metricsFactory = metricsFactory;
-- this.rxTaskScheduler = rxTaskScheduler;
- this.actorSystemManager = actorSystemManager;
- this.uniqueValuesService = uniqueValuesService;
- this.cassandraConfig = cassandraConfig;
++ this.keyspace = keyspace;
++ this.metricsFactory = metricsFactory;
++ this.rxTaskScheduler = rxTaskScheduler;
++ this.actorSystemManager = actorSystemManager;
++ this.uniqueValuesService = uniqueValuesService;
++ this.cassandraConfig = cassandraConfig;
}
+
@Override
public EntityCollectionManager createCollectionManager(ApplicationScope applicationScope) {
Preconditions.checkNotNull(applicationScope);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/37972a2e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 6a04c7c,06b086c..9dce7ef
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -19,44 -19,43 +19,32 @@@
package org.apache.usergrid.persistence.collection.impl;
--import java.util.ArrayList;
--import java.util.Collection;
--import java.util.Collections;
--import java.util.Iterator;
--import java.util.List;
--import java.util.UUID;
--
++import com.codahale.metrics.Timer;
++import com.google.common.base.Preconditions;
++import com.google.inject.Inject;
++import com.google.inject.assistedinject.Assisted;
++import com.netflix.astyanax.Keyspace;
++import com.netflix.astyanax.MutationBatch;
++import com.netflix.astyanax.connectionpool.OperationResult;
++import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
++import com.netflix.astyanax.model.ColumnFamily;
import com.netflix.astyanax.model.ConsistencyLevel;
-import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator;
-import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.FieldSet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.VersionSet;
++import com.netflix.astyanax.model.CqlResult;
++import com.netflix.astyanax.serializers.StringSerializer;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
- import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
- import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
-
- import org.apache.usergrid.persistence.collection.EntityCollectionManager;
- import org.apache.usergrid.persistence.collection.EntitySet;
- import org.apache.usergrid.persistence.collection.FieldSet;
- import org.apache.usergrid.persistence.collection.MvccEntity;
- import org.apache.usergrid.persistence.collection.MvccLogEntry;
- import org.apache.usergrid.persistence.collection.VersionSet;
++import org.apache.usergrid.persistence.collection.*;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.UniqueCleanup;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.VersionCompact;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
--import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteUniqueVerify;
--import org.apache.usergrid.persistence.collection.serialization.MvccEntitySerializationStrategy;
--import org.apache.usergrid.persistence.collection.serialization.MvccLogEntrySerializationStrategy;
--import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
--import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
--import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
--import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
++import org.apache.usergrid.persistence.collection.mvcc.stage.write.*;
++import org.apache.usergrid.persistence.collection.serialization.*;
++import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator;
import org.apache.usergrid.persistence.collection.serialization.impl.MinMaxLogEntryIterator;
import org.apache.usergrid.persistence.collection.serialization.impl.MutableFieldSet;
++import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
++import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
@@@ -68,22 -67,22 +56,13 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
--
--import com.codahale.metrics.Timer;
--import com.google.common.base.Preconditions;
--import com.google.inject.Inject;
--import com.google.inject.assistedinject.Assisted;
--import com.netflix.astyanax.Keyspace;
--import com.netflix.astyanax.MutationBatch;
--import com.netflix.astyanax.connectionpool.OperationResult;
--import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
--import com.netflix.astyanax.model.ColumnFamily;
--import com.netflix.astyanax.model.CqlResult;
--import com.netflix.astyanax.serializers.StringSerializer;
--
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
import rx.Observable;
import rx.Subscriber;
++import java.util.*;
++
/**
* Simple implementation. Should perform writes, delete and load.
@@@ -125,31 -125,20 +105,35 @@@ public class EntityCollectionManagerImp
private final ApplicationScope applicationScope;
private final RxTaskScheduler rxTaskScheduler;
+ private final UniqueValuesService uniqueValuesService;
+ private final ActorSystemManager actorSystemManager;
+
@Inject
- public EntityCollectionManagerImpl( final WriteStart writeStart,
- final WriteUniqueVerify writeVerifyUnique,
- public EntityCollectionManagerImpl( final WriteStart writeStart, final WriteUniqueVerify writeVerifyUnique,
-- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit,
- final RollbackAction rollback,
- final MarkStart markStart,
- final MarkCommit markCommit,
- final UniqueCleanup uniqueCleanup,
- final VersionCompact versionCompact,
- final WriteCommit writeCommit, final RollbackAction rollback,
- final MarkStart markStart, final MarkCommit markCommit,
- final UniqueCleanup uniqueCleanup, final VersionCompact versionCompact,
-- final MvccEntitySerializationStrategy entitySerializationStrategy,
-- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
-- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- final Keyspace keyspace,
- final MetricsFactory metricsFactory,
- final SerializationFig serializationFig,
- final RxTaskScheduler rxTaskScheduler,
- ActorSystemManager actorSystemManager,
- UniqueValuesService uniqueValuesService,
- @Assisted final ApplicationScope applicationScope ) {
- final Keyspace keyspace, final MetricsFactory metricsFactory,
- final SerializationFig serializationFig, final RxTaskScheduler rxTaskScheduler,
- @Assisted final ApplicationScope applicationScope,
- final CassandraConfig cassandraConfig) {
++ public EntityCollectionManagerImpl(
++ final WriteStart writeStart,
++ final WriteUniqueVerify writeVerifyUnique,
++ final WriteOptimisticVerify writeOptimisticVerify,
++ final WriteCommit writeCommit,
++ final RollbackAction rollback,
++ final MarkStart markStart,
++ final MarkCommit markCommit,
++ final UniqueCleanup uniqueCleanup,
++ final VersionCompact versionCompact,
++
++ final MvccEntitySerializationStrategy entitySerializationStrategy,
++ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
++ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
++
++ final Keyspace keyspace,
++ final MetricsFactory metricsFactory,
++ final SerializationFig serializationFig,
++ final RxTaskScheduler rxTaskScheduler,
++ final ActorSystemManager actorSystemManager,
++ final UniqueValuesService uniqueValuesService,
++ final CassandraConfig cassandraConfig,
++ @Assisted final ApplicationScope applicationScope ) {
+
this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
this.entitySerializationStrategy = entitySerializationStrategy;
this.uniqueCleanup = uniqueCleanup;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/37972a2e/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 1f86440,b36cb79..a3565ea
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@@ -170,11 -121,13 +170,11 @@@ public class WriteUniqueVerify implemen
try {
- // loading will retrieve the oldest unique value entry for the field
// don't use read repair on this pre-write check
- // use CL ALL as consistency is extremely important here, more so than performance
- UniqueValueSet set = uniqueValueStrat.load(scope, ConsistencyLevel.CL_ALL,
+ // stronger consistency is extremely important here, more so than performance
+ UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getConsistentReadCL(),
written.getEntityId().getType(), Collections.singletonList(written.getField()), false);
-
set.forEach(uniqueValue -> {
if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
[44/50] usergrid git commit: Increase test coverage of actorsystem
module
Posted by mr...@apache.org.
Increase test coverage of actorsystem module
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/31b20404
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/31b20404
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/31b20404
Branch: refs/heads/master
Commit: 31b20404da508f9fa35ae24d2f02381209944406
Parents: 47b7615
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 16:21:36 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 16:21:36 2016 -0400
----------------------------------------------------------------------
stack/corepersistence/actorsystem/pom.xml | 7 +++++
.../actorsystem/ActorServiceServiceTest.java | 33 +++++++++++++++-----
2 files changed, 32 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/31b20404/stack/corepersistence/actorsystem/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/pom.xml b/stack/corepersistence/actorsystem/pom.xml
index 85c0d60..b77f90f 100644
--- a/stack/corepersistence/actorsystem/pom.xml
+++ b/stack/corepersistence/actorsystem/pom.xml
@@ -94,6 +94,13 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/31b20404/stack/corepersistence/actorsystem/src/test/java/org/apache/usergrid/persistence/actorsystem/ActorServiceServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/test/java/org/apache/usergrid/persistence/actorsystem/ActorServiceServiceTest.java b/stack/corepersistence/actorsystem/src/test/java/org/apache/usergrid/persistence/actorsystem/ActorServiceServiceTest.java
index a12c5e1..7ac7b12 100644
--- a/stack/corepersistence/actorsystem/src/test/java/org/apache/usergrid/persistence/actorsystem/ActorServiceServiceTest.java
+++ b/stack/corepersistence/actorsystem/src/test/java/org/apache/usergrid/persistence/actorsystem/ActorServiceServiceTest.java
@@ -24,11 +24,17 @@ import org.apache.usergrid.persistence.core.test.UseModules;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import java.util.concurrent.atomic.AtomicBoolean;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.verify;
+
@RunWith( ITRunner.class )
@UseModules( ActorSystemModule.class )
@@ -38,20 +44,31 @@ public class ActorServiceServiceTest {
@Inject
ActorSystemFig actorSystemFig;
- private static AtomicBoolean startedAkka = new AtomicBoolean( false );
+ @Inject
+ ActorSystemManager actorSystemManager;
- @Before
- public void initAkka() {
- if ( !startedAkka.getAndSet( true ) ) {
- }
- }
+ private static AtomicBoolean startedAkka = new AtomicBoolean( false );
@Test
public void testBasicOperation() throws Exception {
- initAkka();
- }
+ RouterProducer routerProducer = Mockito.mock( RouterProducer.class );
+ actorSystemManager.registerRouterProducer( routerProducer );
+
+ actorSystemManager.registerMessageType( String.class, "/users/path" );
+ actorSystemManager.registerMessageType( Integer.class, "/users/path" );
+ actorSystemManager.registerMessageType( Long.class, "/users/path" );
+
+ actorSystemManager.start( "localhost", 2770, "us-east" );
+ actorSystemManager.waitForClientActor();
+
+ verify( routerProducer ).createClusterSingletonManager( any() );
+ verify( routerProducer ).createClusterSingletonProxy( any(), eq("io") );
+ verify( routerProducer ).createLocalSystemActors( any() );
+ verify( routerProducer ).addConfiguration( any() );
+
+ }
}
[34/50] usergrid git commit: Fix tomcat dependency versions.
Posted by mr...@apache.org.
Fix tomcat dependency versions.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3e65e78c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3e65e78c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3e65e78c
Branch: refs/heads/master
Commit: 3e65e78c33ca47404905cea8558d9e0e2800241f
Parents: 4613e09
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Jul 6 17:25:56 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Jul 6 17:25:56 2016 -0700
----------------------------------------------------------------------
stack/pom.xml | 3 ++-
stack/test-utils/pom.xml | 5 +++++
2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3e65e78c/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 1bf1f92..5fd0817 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -118,6 +118,7 @@
<slf4j-version>1.7.2</slf4j-version>
<snakeyaml-version>1.9</snakeyaml-version>
<tomcat-version>8.5.3</tomcat-version>
+ <tomcat-logging-version>8.5.2</tomcat-logging-version>
<antlr.version>3.4</antlr.version>
<tika.version>1.4</tika.version>
<mockito.version>1.10.8</mockito.version>
@@ -908,7 +909,7 @@
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-logging-juli</artifactId>
- <version>${tomcat-version}</version>
+ <version>${tomcat-logging-version}</version>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3e65e78c/stack/test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/stack/test-utils/pom.xml b/stack/test-utils/pom.xml
index c104f0a..b460198 100644
--- a/stack/test-utils/pom.xml
+++ b/stack/test-utils/pom.xml
@@ -245,26 +245,31 @@
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
+ <version>${tomcat-version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-logging-juli</artifactId>
+ <version>${tomcat-logging-version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-jasper</artifactId>
+ <version>${tomcat-version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-jasper-el</artifactId>
+ <version>${tomcat-version}</version>
</dependency>
<dependency>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-jsp-api</artifactId>
+ <version>${tomcat-version}</version>
</dependency>
</dependencies>
[22/50] usergrid git commit: Fix issue with checking pre-write unique
value violations.
Posted by mr...@apache.org.
Fix issue with checking pre-write unique value violations.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3bba4259
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3bba4259
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3bba4259
Branch: refs/heads/master
Commit: 3bba4259bdf17472760eac33de487a972f7058c2
Parents: 61aa037
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jul 1 08:45:19 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jul 1 08:45:19 2016 -0700
----------------------------------------------------------------------
.../collection/mvcc/stage/write/WriteUniqueVerify.java | 2 +-
.../impl/UniqueValueSerializationStrategyImpl.java | 7 +++----
2 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bba4259/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 7f32888..538160a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -156,7 +156,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
}
// only build the batch statement if we don't have a violation for the field
- if( preWriteUniquenessViolations.get(field.getName()) != null) {
+ if( preWriteUniquenessViolations.get(field.getName()) == null) {
// use TTL in case something goes wrong before entity is finally committed
final MutationBatch mb = uniqueValueStrat.write(scope, written, serializationFig.getTimeout());
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bba4259/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index d7b1be9..8f304f3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -223,10 +223,9 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
if ( logger.isTraceEnabled() ) {
- logger.trace( "Writing unique value version={} name={} value={} ",
- uniqueValue.getEntityVersion(), uniqueValue.getField().getName(),
- uniqueValue.getField().getValue()
- );
+ logger.trace( "Building batch statement for unique value entity={} version={} name={} value={} ",
+ uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion(),
+ uniqueValue.getField().getName(), uniqueValue.getField().getValue() );
}