You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2016/07/07 16:11:34 UTC
[01/16] usergrid git commit: Add additional logging during unique
value read repair.
Repository: usergrid
Updated Branches:
refs/heads/usergrid-1268-akka-211 f0c9fd4bd -> e6c59ee3f
Add additional logging during unique value read repair.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b0c30593
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b0c30593
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b0c30593
Branch: refs/heads/usergrid-1268-akka-211
Commit: b0c30593a24d23fedbc16e033b91fb8cb1d999ea
Parents: 8242878
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Jun 30 20:36:01 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Jun 30 20:36:01 2016 -0700
----------------------------------------------------------------------
.../impl/UniqueValueSerializationStrategyImpl.java | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b0c30593/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index 4456123..ce1f20e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -335,7 +335,9 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
// do nothing, only versions can be newer and we're not worried about newer versions of same entity
if (logger.isTraceEnabled()) {
- logger.trace("Candidate unique value is equal to the current unique value");
+ logger.trace("Current unique value [{}={}] entry has UUID [{}] equal to candidate UUID [{}]",
+ field.getName(), field.getValue().toString(), uniqueValue.getEntityId().getUuid(),
+ candidates.get(candidates.size() -1));
}
// update candidate w/ latest version
@@ -365,7 +367,8 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
candidates.clear();
if (logger.isTraceEnabled()) {
- logger.trace("Updating candidate to entity id [{}] and entity version [{}]",
+ logger.trace("Updating candidate unique value [{}={}] to entity id [{}] and " +
+ "entity version [{}]", field.getName(), field.getValue().toString(),
uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion());
}
@@ -390,6 +393,10 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
}
// take the last candidate ( should be the latest version) and add to the result set
+ if(logger.isTraceEnabled()){
+ logger.trace("Adding unique value [{}] to response set.",
+ candidates.get(candidates.size() -1));
+ }
uniqueValueSet.addValue(candidates.get(candidates.size() -1));
}
[12/16] usergrid git commit: Merge branch 'release-2.1.1' of
https://git-wip-us.apache.org/repos/asf/usergrid into usergrid-1268-akka-211
Posted by sn...@apache.org.
Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into usergrid-1268-akka-211
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/670248dd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/670248dd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/670248dd
Branch: refs/heads/usergrid-1268-akka-211
Commit: 670248ddaff65db78794bbdceb3c569b0543f6d3
Parents: 557ea91 8982ddd
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Jul 5 12:57:24 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Jul 5 12:57:24 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 44 +++++---
.../persistence/collection/FieldSet.java | 12 +++
.../impl/EntityCollectionManagerImpl.java | 27 +----
.../mvcc/stage/write/WriteUniqueVerify.java | 19 +++-
.../serialization/SerializationFig.java | 4 -
.../serialization/impl/MutableFieldSet.java | 12 +++
.../UniqueValueSerializationStrategyImpl.java | 48 +++++++--
.../usergrid/tools/UniqueValueScanner.java | 105 +++++++++++--------
8 files changed, 176 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/670248dd/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/670248dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/670248dd/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 985137b,01e6621..e7ba967
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@@ -170,10 -121,13 +170,11 @@@ public class WriteUniqueVerify implemen
try {
- // loading will retrieve the oldest unique value entry for the field
// don't use read repair on this pre-write check
- UniqueValueSet set = uniqueValueStrat.load(scope, written.getEntityId().getType(),
- Collections.singletonList(written.getField()), false);
+ // use CL ALL as consistency is extremely important here, more so than performance
+ UniqueValueSet set = uniqueValueStrat.load(scope, ConsistencyLevel.CL_ALL,
+ written.getEntityId().getType(), Collections.singletonList(written.getField()), false);
-
set.forEach(uniqueValue -> {
if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
[08/16] usergrid git commit: Clean up unique value scanner and let it
get consistently level from the typically read CL property.
Posted by sn...@apache.org.
Clean up unique value scanner and let it get consistently level from the typically read CL property.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ada914ac
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ada914ac
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ada914ac
Branch: refs/heads/usergrid-1268-akka-211
Commit: ada914ac32ade7396f1b4e0f33d147ccbf328137
Parents: 3bba425
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jul 1 14:55:08 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jul 1 14:55:08 2016 -0700
----------------------------------------------------------------------
.../usergrid/tools/UniqueValueScanner.java | 105 +++++++++++--------
1 file changed, 60 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ada914ac/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
index 182e692..68f366b 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueValueScanner.java
@@ -18,9 +18,11 @@ package org.apache.usergrid.tools;
import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ConsistencyLevel;
import com.netflix.astyanax.util.RangeBuilder;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
@@ -131,6 +133,7 @@ public class UniqueValueScanner extends ToolBase {
}
logger.info("Staring Tool: UniqueValueScanner");
+ logger.info("Using Cassandra consistency level: {}", System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM"));
keyspace = injector.getInstance(com.netflix.astyanax.Keyspace.class);
@@ -142,6 +145,8 @@ public class UniqueValueScanner extends ToolBase {
String entityType = line.getOptionValue(ENTITY_TYPE_ARG);
String entityName = line.getOptionValue(ENTITY_NAME_ARG);
+ AtomicInteger count = new AtomicInteger(0);
+
if (entityName != null && !entityName.isEmpty()) {
if(appToFilter == null){
@@ -157,11 +162,11 @@ public class UniqueValueScanner extends ToolBase {
logger.info("Running entity unique load only");
- //do stuff
+ //do stuff w/o read repair
UniqueValueSet uniqueValueSet = uniqueValueSerializationStrategy.load(
new ApplicationScopeImpl( new SimpleId(appToFilter, "application" ) ),
- entityType,
- Collections.singletonList(new StringField( fieldType, entityName) ));
+ ConsistencyLevel.valueOf(System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")), entityType,
+ Collections.singletonList(new StringField( fieldType, entityName) ), false);
StringBuilder stringBuilder = new StringBuilder();
@@ -194,89 +199,99 @@ public class UniqueValueScanner extends ToolBase {
try {
rows = keyspace.prepareQuery(CF_UNIQUE_VALUES)
+ .setConsistencyLevel(ConsistencyLevel.valueOf(System.getProperty("usergrid.read.cl", "CL_LOCAL_QUORUM")))
.getAllRows()
- .withColumnRange(new RangeBuilder().build())
+ .withColumnRange(new RangeBuilder().setLimit(1000).build())
.execute().getResult().iterator();
} catch (ConnectionException e) {
+ logger.error("Error connecting to cassandra", e);
}
UUID finalAppToFilter = appToFilter;
- rows.forEachRemaining(row -> {
- final String fieldName = row.getKey().getKey().getField().getName();
- final String fieldValue = row.getKey().getKey().getField().getValue().toString();
- final String scopeType = row.getKey().getScope().getType();
- final UUID scopeUUID = row.getKey().getScope().getUuid();
+ if( rows != null) {
+ rows.forEachRemaining(row -> {
+ count.incrementAndGet();
- if (!fieldName.equalsIgnoreCase(fieldType) ||
- (finalAppToFilter != null && !finalAppToFilter.equals(scopeUUID))
- ) {
- // do nothing
+ if(count.get() % 1000 == 0 ){
+ logger.info("Scanned {} rows in {}", count.get(), CF_UNIQUE_VALUES.getName());
+ }
- } else {
+ final String fieldName = row.getKey().getKey().getField().getName();
+ final String fieldValue = row.getKey().getKey().getField().getValue().toString();
+ final String scopeType = row.getKey().getScope().getType();
+ final UUID scopeUUID = row.getKey().getScope().getUuid();
- if (em == null && finalAppToFilter.equals(scopeUUID)) {
- em = emf.getEntityManager(scopeUUID);
- }
- // if we have more than 1 column, let's check for a duplicate
- if(row.getColumns().size() > 1) {
+ if (!fieldName.equalsIgnoreCase(fieldType) ||
+ (finalAppToFilter != null && !finalAppToFilter.equals(scopeUUID))
+ ) {
+ // do nothing
- final List<EntityVersion> values = new ArrayList<>(row.getColumns().size());
+ } else {
- Iterator<Column<EntityVersion>> columns = row.getColumns().iterator();
- columns.forEachRemaining(column -> {
+ // if we have more than 1 column, let's check for a duplicate
+ if (row.getColumns() != null && row.getColumns().size() > 1) {
+ final List<EntityVersion> values = new ArrayList<>(row.getColumns().size());
- final EntityVersion entityVersion = column.getName();
+ Iterator<Column<EntityVersion>> columns = row.getColumns().iterator();
+ columns.forEachRemaining(column -> {
+ final EntityVersion entityVersion = column.getName();
- logger.trace(
- scopeType + ": " + scopeUUID + ", " +
- fieldName + ": " + fieldValue + ", " +
- "entity type: " + entityVersion.getEntityId().getType() + ", " +
- "entity uuid: " + entityVersion.getEntityId().getUuid()
- );
+ logger.trace(
+ scopeType + ": " + scopeUUID + ", " +
+ fieldName + ": " + fieldValue + ", " +
+ "entity type: " + entityVersion.getEntityId().getType() + ", " +
+ "entity uuid: " + entityVersion.getEntityId().getUuid()
+ );
- if (entityType != null &&
- entityVersion.getEntityId().getType().equalsIgnoreCase(entityType)
- ) {
- // add the first value into the list
- if(values.size() == 0 ) {
+ if (entityType != null &&
+ entityVersion.getEntityId().getType().equalsIgnoreCase(entityType)
+ ) {
- values.add(entityVersion);
+ // add the first value into the list
+ if (values.size() == 0) {
+ values.add(entityVersion);
- }else{
- if( !values.get(0).getEntityId().getUuid().equals(entityVersion.getEntityId().getUuid())){
+ } else {
- values.add(entityVersion);
+ if (!values.get(0).getEntityId().getUuid().equals(entityVersion.getEntityId().getUuid())) {
+
+ values.add(entityVersion);
- logger.error("Duplicate found for field [{}={}]. Entry 1: [{}], Entry 2: [{}]",
- fieldName, fieldValue, values.get(0).getEntityId(), entityVersion.getEntityId());
+ logger.error("Duplicate found for field [{}={}]. Entry 1: [{}], Entry 2: [{}]",
+ fieldName, fieldValue, values.get(0).getEntityId(), entityVersion.getEntityId());
+
+ }
}
+
}
+ });
+ }
+ }
- }
- });
- }
- }
+ });
+ }else{
+ logger.warn("No rows returned from table: {}", CF_UNIQUE_VALUES.getName());
- });
+ }
}
}
[07/16] usergrid git commit: Fix ports in REST tests too.
Posted by sn...@apache.org.
Fix ports in REST tests too.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/557ea914
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/557ea914
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/557ea914
Branch: refs/heads/usergrid-1268-akka-211
Commit: 557ea91433dc3ffd8b78b89888fa0349226a860a
Parents: 5ddb1cd
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jul 1 16:41:47 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jul 1 16:41:47 2016 -0400
----------------------------------------------------------------------
.../src/test/resources/usergrid.properties | 3 ++-
.../test/resources/usergrid-custom-test.properties | 16 +++++++---------
2 files changed, 9 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/557ea914/stack/corepersistence/collection/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid.properties b/stack/corepersistence/collection/src/test/resources/usergrid.properties
index 8c999e2..ebc2ad4 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid.properties
@@ -20,8 +20,9 @@
# This property is required to be set and cannot be defaulted anywhere
usergrid.cluster_name=usergrid
-usergrid.cluster.hostname=localhost
+
usergrid.cluster.enabled=true
+usergrid.cluster.hostname=localhost
usergrid.cluster.region.local=us-east
usergrid.cluster.region.list=us-east
usergrid.cluster.seeds=us-east\:localhost
http://git-wip-us.apache.org/repos/asf/usergrid/blob/557ea914/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 31c6604..f20f1e5 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -56,14 +56,12 @@ usergrid.cluster_name=usergrid
usergrid.graph.shard.size=100
usergrid.graph.shard.repair.chance=1.0
-collection.akka.enabled=true
-collection.akka.hostname=localhost
-collection.akka.region=us-east
-usergrid.queue.regionList=us-east
-collection.akka.authoritative.region=us-east
-collection.akka.region.seeds=us-east\:localhost\:2551
+usergrid.cluster.enabled=true
+usergrid.cluster.hostname=localhost
+usergrid.cluster.region.local=us-east
+usergrid.cluster.region.list=us-east
+usergrid.cluster.seeds=us-east\:localhost
-collection.akka.uniquevalue.actors=300
-collection.akka.uniquevalue.cache.ttl=10
-collection.akka.uniquevalue.reservation.ttl=10
+collection.uniquevalues.actors=300
+collection.uniquevalues.authoritative.region=us-east
[14/16] usergrid git commit: Fixes to tests and test properties.
Posted by sn...@apache.org.
Fixes to tests and test properties.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/996bf09f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/996bf09f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/996bf09f
Branch: refs/heads/usergrid-1268-akka-211
Commit: 996bf09fbc33bba3c27a26143880ceb100a38936
Parents: db6950b
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 12:09:43 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 12:09:43 2016 -0400
----------------------------------------------------------------------
.../src/test/resources/usergrid-test.properties | 15 ++
.../corepersistence/index/IndexServiceTest.java | 70 ++++---
.../collection/EntityCollectionManagerIT.java | 45 +++--
.../mvcc/stage/delete/MarkCommitTest.java | 3 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 2 +-
.../UniqueValuesServiceDeleteTest.java | 185 +++++++++++++++++++
.../src/test/resources/usergrid.properties | 3 +
.../org/apache/usergrid/rest/UniqueCatsIT.java | 2 +-
.../rest/applications/ApplicationDeleteIT.java | 2 +-
9 files changed, 282 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/config/src/test/resources/usergrid-test.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/test/resources/usergrid-test.properties b/stack/config/src/test/resources/usergrid-test.properties
index 29bb332..368c585 100644
--- a/stack/config/src/test/resources/usergrid-test.properties
+++ b/stack/config/src/test/resources/usergrid-test.properties
@@ -83,6 +83,21 @@ usergrid.version.properties=1.0.0
# build number for display
usergrid.version.build=0.1
+
+# --- Start: Usergrid cluster/actor system settings
+
+usergrid.cluster.enabled=true
+usergrid.cluster.hostname=localhost
+usergrid.cluster.region.local=us-east
+usergrid.cluster.region.list=us-east
+usergrid.cluster.seeds=us-east\:localhost
+
+collection.uniquevalues.actors=300
+collection.uniquevalues.authoritative.region=us-east
+
+# --- End: Usergrid cluster/actor system settings
+
+
usergrid.counter.batch.interval=10
#usergrid.auth.token_secret_salt=super secret token value
#usergrid.auth.token_expires_from_last_use=false
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
index dab14ed..fba6e7c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/IndexServiceTest.java
@@ -20,62 +20,54 @@
package org.apache.usergrid.corepersistence.index;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.graph.MarkedEdge;
-import org.apache.usergrid.persistence.index.impl.IndexProducer;
-import org.junit.Before;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-
+import com.google.inject.Inject;
+import net.jcip.annotations.NotThreadSafe;
import org.apache.usergrid.corepersistence.TestIndexModule;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueActor;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.test.UseModules;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.persistence.index.CandidateResults;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.IndexFig;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.impl.EsRunner;
-import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.index.impl.IndexOperation;
+import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.google.inject.Inject;
-
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
import rx.Observable;
import rx.schedulers.Schedulers;
+import java.util.*;
+
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionEdge;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getApplicationScope;
import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
@RunWith( EsRunner.class )
@UseModules( { TestIndexModule.class } )
+@NotThreadSafe//anything that changes the system version state is not safe to be run concurrently
public class IndexServiceTest {
@Inject
public IndexService indexService;
-
@Inject
public GraphManagerFactory graphManagerFactory;
@@ -98,12 +90,39 @@ public class IndexServiceTest {
public ApplicationScope applicationScope;
+ @Inject
+ ActorSystemManager actorSystemManager;
+
+ @Inject
+ UniqueValuesService uniqueValuesService;
+
+
+ private static Map<Integer, Boolean> startedAkka = new HashMap<>();
+
+
+ protected synchronized void initAkka(
+ int port, ActorSystemManager actorSystemManager, UniqueValuesService uniqueValuesService ) {
+
+ if ( startedAkka.get(port) == null ) {
+
+ actorSystemManager.registerRouterProducer( uniqueValuesService );
+ actorSystemManager.registerMessageType( UniqueValueActor.Request.class, "/user/uvProxy" );
+ actorSystemManager.registerMessageType( UniqueValueActor.Reservation.class, "/user/uvProxy" );
+ actorSystemManager.registerMessageType( UniqueValueActor.Cancellation.class, "/user/uvProxy" );
+ actorSystemManager.registerMessageType( UniqueValueActor.Confirmation.class, "/user/uvProxy" );
+ actorSystemManager.start( "localhost", port, "us-east" );
+ actorSystemManager.waitForClientActor();
+
+ startedAkka.put( port, true );
+ }
+ }
+
@Before
public void setup() {
applicationScope = getApplicationScope( UUIDGenerator.newTimeUUID() );
-
graphManager = graphManagerFactory.createEdgeManager( applicationScope );
+ initAkka( 2555, actorSystemManager, uniqueValuesService );
}
@@ -140,7 +159,6 @@ public class IndexServiceTest {
@Test( )
public void testSingleCollectionConnection() throws InterruptedException {
-
ApplicationScope applicationScope =
new ApplicationScopeImpl( new SimpleId( UUID.randomUUID(), "application" ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index ceda1a8..dfff51c 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -49,6 +49,8 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import rx.Observable;
import java.util.ArrayList;
@@ -63,6 +65,7 @@ import static org.junit.Assert.*;
@RunWith( ITRunner.class )
@UseModules( TestCollectionModule.class )
public class EntityCollectionManagerIT extends AbstractUniqueValueTest {
+ private static final Logger logger = LoggerFactory.getLogger( EntityCollectionManagerIT.class );
@Inject
private EntityCollectionManagerFactory factory;
@@ -137,7 +140,7 @@ public class EntityCollectionManagerIT extends AbstractUniqueValueTest {
}
catch ( Exception ex ) {
WriteUniqueVerifyException e = ( WriteUniqueVerifyException ) ex;
- assertEquals( 1, e.getVioliations().size() );
+ assertEquals( 1, e.getViolations().size() );
}
}
}
@@ -319,66 +322,78 @@ public class EntityCollectionManagerIT extends AbstractUniqueValueTest {
@Test
public void writeAndGetField2X() throws InterruptedException {
- ApplicationScope collectionScope1 = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+ // create entity with unique testField
+ ApplicationScope collectionScope1 = new ApplicationScopeImpl( new SimpleId( "organization" ) );
final Id entityId = new SimpleId( "test" );
Entity firstInstance = new Entity( entityId );
Field firstField = new StringField( "testField", "unique", true );
firstInstance.setField( firstField );
EntityCollectionManager manager = factory.createCollectionManager( collectionScope1 );
-
Observable<Entity> observable = manager.write( firstInstance, null );
-
Entity createReturned = observable.toBlocking().lastOrDefault( null );
-
assertNotNull( "Id was assigned", createReturned.getId() );
assertNotNull( "Version was assigned", createReturned.getVersion() );
+ // get entity via that unique field, should get correct entity
+
final Id existingId = manager.getIdField( firstInstance.getId().getType(), firstField )
.toBlocking().lastOrDefault( null );
assertNotNull( existingId );
assertEquals( firstInstance.getId(), existingId );
+ // get entity via bogus unique field that does not exist, should get null
+
Field fieldNull = new StringField( "testFieldNotThere", "uniquely", true );
final Id noId = manager.getIdField( firstInstance.getId().getType(), fieldNull )
.toBlocking().lastOrDefault( null );
assertNull( noId );
+ // ensure we clean up
- //ensure we clean up
+ // set a different unique field to the entity we created above
+ // this should effectively remove the original unique testField that we created above
Entity secondInstance = new Entity( entityId );
Field secondField = new StringField( firstField.getName(), "unique2", true );
secondInstance.setField( secondField );
Observable<Entity> observableSecond = manager.write( secondInstance, null );
-
Entity createReturnedSecond = observableSecond.toBlocking().lastOrDefault( null );
-
assertNotNull( "Id was assigned", createReturnedSecond.getId() );
assertNotNull( "Version was assigned", createReturnedSecond.getVersion() );
assertNotEquals( "Versions should not be equal",
createReturned.getVersion(), createReturnedSecond.getVersion() );
- //sanity check, get the entity to ensure it's the right version
+ // sanity check, get the entity to ensure it's the right version
final Entity loadedVersion = manager.load( entityId ).toBlocking().last();
assertEquals(entityId, loadedVersion.getId());
assertEquals(createReturnedSecond.getVersion(), loadedVersion.getVersion());
- //give clean time to run. need to finish the todo below
- Thread.sleep( 2000 );
+ // give clean time to run. need to finish the todo below
- //TODO, we need to implement verify and repair on this
- final Id idFirst = manager.getIdField( firstInstance.getId().getType(), firstField )
- .toBlocking().lastOrDefault( null );
- assertNull(idFirst);
+ Id idFirst = null;
+ int retries = 0;
+ while ( retries++ < 20 ) {
+
+ //TODO, we need to implement verify and repair on this
+ idFirst = manager.getIdField( firstInstance.getId().getType(), firstField )
+ .toBlocking().lastOrDefault( null );
+ if ( idFirst == null ) {
+ break;
+ }
+
+ logger.error("Clean no run yet, waiting ({})", retries);
+ Thread.sleep( 2000 );
+ }
+ assertNull(idFirst);
final Id idSecond = manager.getIdField( secondInstance.getId().getType(), secondField )
.toBlocking().lastOrDefault( null );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
index a0ee6be..89169ac 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
@@ -120,7 +120,8 @@ public class MarkCommitTest extends AbstractMvccEntityStageTest {
.thenReturn( entityMutation );
- new MarkCommit( logStrategy, mvccEntityStrategy, uniqueValueSerializationStrategy, serializationFig, keyspace ).call( event );
+ new MarkCommit( logStrategy, mvccEntityStrategy, uniqueValueSerializationStrategy, serializationFig,
+ null, null, null, keyspace ).call( event );
//TODO: This doesn't assert anything, this needs fixed (should be a fail technically)
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
index c70cd1a..401d23e 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
@@ -123,7 +123,7 @@ public class WriteUniqueVerifyIT extends AbstractUniqueValueTest {
} catch ( Exception ex ) {
WriteUniqueVerifyException e = (WriteUniqueVerifyException)ex;
- assertTrue( !e.getVioliations().isEmpty() );
+ assertTrue( !e.getViolations().isEmpty() );
}
// ensure we can update original entity without error
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
new file mode 100644
index 0000000..3f58834
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.collection.uniquevalues;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Multimaps;
+import com.google.inject.Inject;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
+import org.apache.usergrid.persistence.collection.AbstractUniqueValueTest;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
+import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
+import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import rx.Observable;
+
+import java.util.Collection;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+
+/**
+ * Test the unique values service.
+ */
+@RunWith( ITRunner.class )
+@UseModules( TestCollectionModule.class )
+public class UniqueValuesServiceDeleteTest extends AbstractUniqueValueTest {
+ private static final Logger logger = LoggerFactory.getLogger( UniqueValuesServiceDeleteTest.class );
+
+ @Inject
+ private EntityCollectionManagerFactory factory;
+
+ @Inject
+ @Rule
+ public MigrationManagerRule migrationManagerRule;
+
+ @Inject
+ ActorSystemFig actorSystemFig;
+
+ @Inject
+ ActorSystemManager actorSystemManager;
+
+ @Inject
+ UniqueValuesService uniqueValuesService;
+
+
+ int numThreads = 6;
+ int poolSize = 5;
+ int numUsers = 100;
+
+
+ @Before
+ public void initAkka() {
+ // each test class needs unique port number
+ initAkka( 2555, actorSystemManager, uniqueValuesService );
+ }
+
+
+ /**
+ * Use multiple threads to attempt to create entities with duplicate usernames.
+ */
+ @Test
+ public void testDuplicatePrevention() throws Exception {
+
+ initAkka();
+
+ final AtomicInteger successCounter = new AtomicInteger( 0 );
+ final AtomicInteger errorCounter = new AtomicInteger( 0 );
+
+ Multimap<String, Entity> usersCreated =
+ generateDuplicateUsers( numUsers, successCounter, errorCounter );
+
+ int userCount = 0;
+ int usernamesWithDuplicates = 0;
+ for ( String username : usersCreated.keySet() ) {
+ Collection<Entity> users = usersCreated.get( username );
+ if ( users.size() > 1 ) {
+ usernamesWithDuplicates++;
+ }
+ userCount++;
+ }
+
+ Assert.assertEquals( 0, usernamesWithDuplicates );
+
+ Assert.assertEquals( numUsers, successCounter.get() );
+ Assert.assertEquals( 0, errorCounter.get() );
+ Assert.assertEquals( numUsers, usersCreated.size() );
+ Assert.assertEquals( numUsers, userCount );
+ }
+
+
+ private Multimap<String, Entity> generateDuplicateUsers(
+ int numUsers, AtomicInteger successCounter, AtomicInteger errorCounter ) {
+
+ ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
+
+ EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ Multimap<String, Entity> usersCreated =
+ Multimaps.synchronizedListMultimap( ArrayListMultimap.create() );
+
+ ExecutorService execService = Executors.newFixedThreadPool( poolSize );
+
+ for (int i = 0; i < numUsers; i++) {
+
+ // multiple threads simultaneously trying to create a user with the same propertyName
+ for (int j = 0; j < numThreads; j++) {
+ String username = "user_" + i;
+
+ execService.submit( () -> {
+
+ try {
+
+ // give entity two unqiue fields username and email
+ Entity newEntity = new Entity( new SimpleId( "user" ) );
+ newEntity.setField( new StringField( "username", username, true ) );
+ newEntity.setField( new StringField( "email", username + "@example.org", true ) );
+
+ Observable<Entity> observable = manager.write( newEntity, null );
+ Entity returned = observable.toBlocking().lastOrDefault( null );
+
+ usersCreated.put( username, newEntity );
+ successCounter.incrementAndGet();
+
+ logger.debug("Created user {}", username);
+
+ } catch ( Throwable t ) {
+ if ( t instanceof WriteUniqueVerifyException) {
+ // we expect lots of these
+ } else {
+ errorCounter.incrementAndGet();
+ logger.error( "Error creating user " + username, t );
+ }
+ }
+
+ } );
+ }
+ }
+ execService.shutdown();
+
+ try {
+ while (!execService.awaitTermination( 60, TimeUnit.SECONDS )) {
+ System.out.println( "Waiting..." );
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return usersCreated;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/corepersistence/collection/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid.properties b/stack/corepersistence/collection/src/test/resources/usergrid.properties
index ebc2ad4..8d615d0 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid.properties
@@ -20,6 +20,7 @@
# This property is required to be set and cannot be defaulted anywhere
usergrid.cluster_name=usergrid
+# --- Start: Usergrid cluster/actor system settings
usergrid.cluster.enabled=true
usergrid.cluster.hostname=localhost
@@ -29,3 +30,5 @@ usergrid.cluster.seeds=us-east\:localhost
collection.uniquevalues.actors=300
collection.uniquevalues.authoritative.region=us-east
+
+# --- End: Usergrid cluster/actor system settings
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
index 0120660..c0209af 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/UniqueCatsIT.java
@@ -49,7 +49,7 @@ import static com.codahale.metrics.MetricRegistry.name;
/**
- * Tests that Catgrid will not allow creation of entities with duplicate names.
+ * Tests that Usergrid will not allow creation of entities with duplicate names.
*
* Intended for use against a production-like cluster, not run during normal JUnit testing.
*
http://git-wip-us.apache.org/repos/asf/usergrid/blob/996bf09f/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
index cc635ed..c1301bb 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
@@ -77,7 +77,7 @@ public class ApplicationDeleteIT extends AbstractRestIT {
ApiResponse appCreateAgainResponse = null;
int retries = 0;
- while ( retries++ < 100 ) {
+ while ( retries++ < 30 ) {
try {
appCreateAgainResponse = clientSetup.getRestClient()
.management().orgs().org( orgName ).app().getTarget()
[02/16] usergrid git commit: Add additional logging during write
unique verification.
Posted by sn...@apache.org.
Add additional logging during write unique verification.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/57231f0d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/57231f0d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/57231f0d
Branch: refs/heads/usergrid-1268-akka-211
Commit: 57231f0d6594916faa459164cabbce9bfb9ffa4d
Parents: b0c3059
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Jun 30 21:05:35 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Jun 30 21:05:35 2016 -0700
----------------------------------------------------------------------
.../collection/mvcc/stage/write/WriteUniqueVerify.java | 12 ++++++++++++
.../impl/UniqueValueSerializationStrategyImpl.java | 3 ++-
2 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/57231f0d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 0ebb6a1..7f32888 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -131,6 +131,18 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
+ if(logger.isTraceEnabled()){
+ logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and" +
+ "entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " +
+ "entity version [{}]",
+ written.getField().getName(),
+ written.getField().getValue().toString(),
+ written.getEntityId().getUuid(),
+ written.getEntityVersion(),
+ uniqueValue.getEntityId().getUuid(),
+ uniqueValue.getEntityVersion());
+ }
+
preWriteUniquenessViolations.put(field.getName(), field);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/57231f0d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index ce1f20e..f37e219 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -358,7 +358,8 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
delete(appScope, candidate).execute();
} catch (ConnectionException e) {
- // do nothing for now
+ logger.error( "Unable to connect to cassandra during duplicate repair of [{}={}]",
+ field.getName(), field.getValue().toString() );
}
});
[11/16] usergrid git commit: Improve consistency when returning
entities/ids loaded from unique valuie index when entity repair ( removal of
stale entities) has been executed.
Posted by sn...@apache.org.
Improve consistency when returning entities/ids loaded from unique valuie index when entity repair ( removal of stale entities) has been executed.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8982ddd3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8982ddd3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8982ddd3
Branch: refs/heads/usergrid-1268-akka-211
Commit: 8982ddd3a12541e988e802c3fc1dd5feca44ad6d
Parents: 02e1619
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Jul 3 15:25:22 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Jul 3 15:25:22 2016 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 44 ++++++++++++++------
.../persistence/collection/FieldSet.java | 12 ++++++
.../impl/EntityCollectionManagerImpl.java | 23 +---------
.../serialization/SerializationFig.java | 4 --
.../serialization/impl/MutableFieldSet.java | 12 ++++++
5 files changed, 57 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 1ed05bd..318bba2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -860,8 +860,6 @@ public class CpEntityManager implements EntityManager {
Timer.Context repairedEntityGet = entGetRepairedEntityTimer.time();
- // TODO: can't we just sub in the getEntityRepair method here
- // so for every read of a uniqueEntityField we can verify it is correct?
StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString());
@@ -877,11 +875,22 @@ public class CpEntityManager implements EntityManager {
return null;
}
- FieldSet fieldSet = fieldSetObservable
- .doOnError( t ->
- logger.error("Unable to retrieve unique values due to: {}", t.getMessage())
- )
- .toBlocking().last();
+ FieldSet fieldSet = fieldSetObservable.toBlocking().last();
+
+ // do a re-load if we know an entity repair was executed
+ // a hit to performance, but we need to return consistent success respones if we're repairing data
+ if(fieldSet.getEntityRepairExecuted()){
+
+ if(logger.isTraceEnabled()){
+ logger.trace("One or more entities were repaired ( removed ) during loading of unique field [{}={}], " +
+ "executing the unique value lookup once more for consistency", uniqueLookupRepairField.getName(),
+ uniqueLookupRepairField.getValue());
+ }
+
+ fieldSet = ecm.getEntitiesFromFields(
+ Inflector.getInstance().singularize( collectionType ),
+ Collections.singletonList(uniqueLookupRepairField), uniqueIndexRepair).toBlocking().last();
+ }
repairedEntityGet.stop();
@@ -912,11 +921,22 @@ public class CpEntityManager implements EntityManager {
return null;
}
- FieldSet fieldSet = fieldSetObservable
- .doOnError( t ->
- logger.error("Unable to retrieve unique values due to: {}", t.getMessage())
- )
- .toBlocking().last();
+ FieldSet fieldSet = fieldSetObservable.toBlocking().last();
+
+ // do a re-load if we know an entity repair was executed
+ // a hit to performance, but we need to return consistent success respones if we're repairing data
+ if(fieldSet.getEntityRepairExecuted()){
+
+ if(logger.isTraceEnabled()){
+ logger.trace("One or more entities were repaired ( removed ) during loading of unique field [{}={}], " +
+ "executing the unique value lookup once more for consistency", uniqueLookupRepairField.getName(),
+ uniqueLookupRepairField.getValue());
+ }
+
+ fieldSet = ecm.getEntitiesFromFields(
+ Inflector.getInstance().singularize( collectionType ),
+ Collections.singletonList(uniqueLookupRepairField), uniqueIndexRepair).toBlocking().last();
+ }
if(fieldSet == null || fieldSet.isEmpty()) {
return null;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
index 43f470d..1fabd5a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java
@@ -45,4 +45,16 @@ public interface FieldSet {
* @return
*/
public boolean isEmpty();
+
+ /**
+ * Return true if there was at least entity repaired ( removed ) during the building of this set.
+ * @return
+ */
+ public boolean getEntityRepairExecuted();
+
+ /**
+ * Return true if there was at least entity repaired ( removed ) during the building of this set.
+ * @return
+ */
+ public void setEntityRepairExecuted(final boolean entityRepairExecuted);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 5bdcf1d..9261b56 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -413,32 +413,11 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
if ( deleteBatch.getRowCount() > 0 ) {
+ response.setEntityRepairExecuted(true);
deleteBatch.execute();
-
- // optionally sleep after read repair as some tasks immediately try to write after the delete
- if ( serializationFig.getReadRepairDelay() > 0 ){
-
- try {
-
- if(logger.isTraceEnabled()) {
- logger.trace("Sleeping {}ms after unique value read repair execution",
- serializationFig.getReadRepairDelay());
- }
-
- Thread.sleep(Math.min(serializationFig.getReadRepairDelay(), 200L));
-
- } catch (InterruptedException e) {
-
- // do nothing if sleep fails; log and continue on
- logger.warn("Sleep during unique value read repair failed.");
- }
-
- }
-
}
-
return response;
}
catch ( ConnectionException e ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 12033fe..bdd71c7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -58,8 +58,4 @@ public interface SerializationFig extends GuicyFig {
@Default( "150" )
int getUniqueVerifyPoolSize();
-
- @Key ( "collection.readrepair.delay" )
- @Default( "0" ) // in milliseconds
- int getReadRepairDelay();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
index df2b850..da874c8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java
@@ -31,6 +31,8 @@ public class MutableFieldSet implements FieldSet {
private final Map<Field<?>, MvccEntity> entities;
+ private boolean readRepairExecuted;
+
public MutableFieldSet( final int expectedSize ) {
this.entities = new HashMap<>( expectedSize );
@@ -60,4 +62,14 @@ public class MutableFieldSet implements FieldSet {
public boolean isEmpty() {
return entities.size() == 0;
}
+
+ @Override
+ public void setEntityRepairExecuted( final boolean readRepairExecuted ){
+ this.readRepairExecuted = readRepairExecuted;
+ }
+
+ @Override
+ public boolean getEntityRepairExecuted(){
+ return readRepairExecuted;
+ }
}
[06/16] usergrid git commit: Fix port settings for tests
Posted by sn...@apache.org.
Fix port settings for tests
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5ddb1cd7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5ddb1cd7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5ddb1cd7
Branch: refs/heads/usergrid-1268-akka-211
Commit: 5ddb1cd7348b075fa16b66683cb626e11b4d0861
Parents: 1b4e117
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jul 1 15:17:27 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jul 1 15:17:27 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 ++
.../actorsystem/ActorSystemManager.java | 7 ++++
.../actorsystem/ActorSystemManagerImpl.java | 6 +++
.../uniquevalues/UniqueValuesServiceImpl.java | 22 ++++++++---
.../collection/AbstractUniqueValueTest.java | 2 +-
.../collection/EntityCollectionManagerIT.java | 2 +-
.../EntityCollectionManagerStressTest.java | 19 +++++++++-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +-
.../uniquevalues/UniqueValuesServiceTest.java | 2 +-
.../src/test/resources/usergrid.properties | 39 +++++++++++++-------
11 files changed, 81 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index fe70569..95b3e9b 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -431,6 +431,9 @@ usergrid.cluster.seeds=default:localhost
# Port used for cluster communications.
usergrid.cluster.port=2551
+# If unspecified then system will use InetAddress.getLocalHost().getHostName()
+usergrid.cluster.hostname=
+
######################### Usergrid Unique Values Validation ##################
#
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
index cdb6caf..893afca 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManager.java
@@ -21,6 +21,8 @@ package org.apache.usergrid.persistence.actorsystem;
import akka.actor.ActorRef;
+import java.util.Set;
+
public interface ActorSystemManager {
/**
@@ -71,6 +73,11 @@ public interface ActorSystemManager {
String getCurrentRegion();
/**
+ * Get all regions known to system.
+ */
+ public Set<String> getRegions();
+
+ /**
* Publish message to all topic subscribers in all regions.
*/
void publishToAllRegions( String topic, Object message, ActorRef sender );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index a79f447..099b875 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -67,12 +67,18 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
private ListMultimap<String, String> seedsByRegion;
+
@Inject
public ActorSystemManagerImpl( ActorSystemFig actorSystemFig ) {
this.actorSystemFig = actorSystemFig;
}
+ public Set<String> getRegions() {
+ return getSeedsByRegion().keySet();
+ }
+
+
/**
* Init Akka ActorSystems and wait for request actors to start.
*/
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
index 352c2e5..fcee8fa 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
@@ -32,6 +32,7 @@ import akka.util.Timeout;
import com.google.inject.Inject;
import com.google.inject.Injector;
import com.google.inject.Singleton;
+import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.actorsystem.GuiceActorProducer;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -92,9 +93,7 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
public void reserveUniqueValues(
ApplicationScope scope, Entity entity, UUID version, String region ) throws UniqueValueException {
- if ( !actorSystemManager.isReady() ) {
- throw new RuntimeException("Unique values service not initialized, no request actors ready");
- }
+ ready();
try {
for (Field field : entity.getFields()) {
@@ -124,9 +123,7 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
public void confirmUniqueValues(
ApplicationScope scope, Entity entity, UUID version, String region ) throws UniqueValueException {
- if ( !actorSystemManager.isReady() ) {
- throw new RuntimeException("Unique values service not initialized, no request actors ready");
- }
+ ready();
try {
for (Field field : entity.getFields()) {
@@ -202,6 +199,19 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
}
+ private void ready() {
+ if ( !actorSystemManager.isReady() ) {
+ throw new RuntimeException("Unique values service not initialized, no request actors ready");
+ }
+
+ if ( !StringUtils.isEmpty( uniqueValuesFig.getAuthoritativeRegion() )) {
+ if ( !actorSystemManager.getRegions().contains( uniqueValuesFig.getAuthoritativeRegion() ) ) {
+ throw new RuntimeException( "Authoritative region not in region list" );
+ }
+ }
+ }
+
+
private void sendUniqueValueRequest(
Entity entity, String region, UniqueValueActor.Request request ) throws UniqueValueException {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
index e7b4450..3bfc48b 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/AbstractUniqueValueTest.java
@@ -40,7 +40,7 @@ public class AbstractUniqueValueTest {
actorSystemManager.registerMessageType( UniqueValueActor.Reservation.class, "/user/uvProxy" );
actorSystemManager.registerMessageType( UniqueValueActor.Cancellation.class, "/user/uvProxy" );
actorSystemManager.registerMessageType( UniqueValueActor.Confirmation.class, "/user/uvProxy" );
- actorSystemManager.start( "127.0.0.1", port, "us-east" );
+ actorSystemManager.start( "localhost", port, "us-east" );
actorSystemManager.waitForClientActor();
startedAkka.put( port, true );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index f8322c6..ceda1a8 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -89,7 +89,7 @@ public class EntityCollectionManagerIT extends AbstractUniqueValueTest {
@Before
public void initAkka() {
- initAkka( 2555, actorSystemManager, uniqueValuesService );
+ initAkka( 2551, actorSystemManager, uniqueValuesService );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
index 7904a42..562a5fc 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerStressTest.java
@@ -22,6 +22,9 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.usergrid.StressTest;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
+import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -52,7 +55,7 @@ import static org.junit.Assert.assertNotNull;
@RunWith(ITRunner.class)
@UseModules(TestCollectionModule.class)
@Category(StressTest.class)
-public class EntityCollectionManagerStressTest {
+public class EntityCollectionManagerStressTest extends AbstractUniqueValueTest {
private static final Logger logger = LoggerFactory.getLogger(
EntityCollectionManagerStressTest.class );
@@ -63,6 +66,20 @@ public class EntityCollectionManagerStressTest {
@Rule
public MigrationManagerRule migrationManagerRule;
+ @Inject
+ ActorSystemManager actorSystemManager;
+
+ @Inject
+ UniqueValuesService uniqueValuesService;
+
+
+ @Before
+ public void initAkka() {
+ // each test class needs unique port number
+ initAkka( 2552, actorSystemManager, uniqueValuesService );
+ }
+
+
@Test
public void writeThousands() {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
index 703d6c2..c70cd1a 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
@@ -85,7 +85,7 @@ public class WriteUniqueVerifyIT extends AbstractUniqueValueTest {
@Before
public void initAkka() {
// each test class needs unique port number
- initAkka( 2551, actorSystemManager, uniqueValuesService );
+ initAkka( 2553, actorSystemManager, uniqueValuesService );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
index 46cfde1..7afba05 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
@@ -77,7 +77,7 @@ public class WriteUniqueVerifyTest extends AbstractUniqueValueTest {
@Before
public void initAkka() {
// each test class needs unique port number
- initAkka( 2552, actorSystemManager, uniqueValuesService );
+ initAkka( 2554, actorSystemManager, uniqueValuesService );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
index 672e3bf..1788a88 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceTest.java
@@ -86,7 +86,7 @@ public class UniqueValuesServiceTest extends AbstractUniqueValueTest {
@Before
public void initAkka() {
// each test class needs unique port number
- initAkka( 2553, actorSystemManager, uniqueValuesService );
+ initAkka( 2555, actorSystemManager, uniqueValuesService );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5ddb1cd7/stack/corepersistence/collection/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid.properties b/stack/corepersistence/collection/src/test/resources/usergrid.properties
index 759a3b3..8c999e2 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid.properties
@@ -1,17 +1,30 @@
-# This property is required to be set and cannot be defaulted anywhere
-usergrid.cluster_name=usergrid
-collection.akka.enabled=true
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
-collection.akka.hostname=localhost
-collection.akka.port=2551
-collection.akka.region=us-east
-usergrid.queue.regionList=us-east
-collection.akka.authoritative.region=us-east
-collection.akka.region.seeds=us-east\:localhost\:2551
+# This property is required to be set and cannot be defaulted anywhere
+usergrid.cluster_name=usergrid
-collection.akka.uniquevalue.actors=300
-collection.akka.uniquevalue.cache.ttl=10
-collection.akka.uniquevalue.reservation.ttl=10
+usergrid.cluster.hostname=localhost
+usergrid.cluster.enabled=true
+usergrid.cluster.region.local=us-east
+usergrid.cluster.region.list=us-east
+usergrid.cluster.seeds=us-east\:localhost
-collection.akka.uniquevalue.region.types
+collection.uniquevalues.actors=300
+collection.uniquevalues.authoritative.region=us-east
[04/16] usergrid git commit: Fix issue with checking pre-write unique
value violations.
Posted by sn...@apache.org.
Fix issue with checking pre-write unique value violations.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3bba4259
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3bba4259
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3bba4259
Branch: refs/heads/usergrid-1268-akka-211
Commit: 3bba4259bdf17472760eac33de487a972f7058c2
Parents: 61aa037
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jul 1 08:45:19 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jul 1 08:45:19 2016 -0700
----------------------------------------------------------------------
.../collection/mvcc/stage/write/WriteUniqueVerify.java | 2 +-
.../impl/UniqueValueSerializationStrategyImpl.java | 7 +++----
2 files changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bba4259/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 7f32888..538160a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -156,7 +156,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
}
// only build the batch statement if we don't have a violation for the field
- if( preWriteUniquenessViolations.get(field.getName()) != null) {
+ if( preWriteUniquenessViolations.get(field.getName()) == null) {
// use TTL in case something goes wrong before entity is finally committed
final MutationBatch mb = uniqueValueStrat.write(scope, written, serializationFig.getTimeout());
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3bba4259/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index d7b1be9..8f304f3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -223,10 +223,9 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
if ( logger.isTraceEnabled() ) {
- logger.trace( "Writing unique value version={} name={} value={} ",
- uniqueValue.getEntityVersion(), uniqueValue.getField().getName(),
- uniqueValue.getField().getValue()
- );
+ logger.trace( "Building batch statement for unique value entity={} version={} name={} value={} ",
+ uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion(),
+ uniqueValue.getField().getName(), uniqueValue.getField().getValue() );
}
[09/16] usergrid git commit: Add more logging to unique value
serialization.
Posted by sn...@apache.org.
Add more logging to unique value serialization.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e0a5d981
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e0a5d981
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e0a5d981
Branch: refs/heads/usergrid-1268-akka-211
Commit: e0a5d9811f7851263336c04302b0208c64573a73
Parents: ada914a
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Jul 3 10:52:31 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Jul 3 10:52:31 2016 -0700
----------------------------------------------------------------------
.../mvcc/stage/write/WriteUniqueVerify.java | 2 +-
.../UniqueValueSerializationStrategyImpl.java | 22 ++++++++++++++++++++
2 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0a5d981/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 538160a..53160dc 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -132,7 +132,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){
if(logger.isTraceEnabled()){
- logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and" +
+ logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and " +
"entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " +
"entity version [{}]",
written.getField().getName(),
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0a5d981/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index 8f304f3..8c1f2d2 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -278,6 +278,12 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
.withColumnRange(new RangeBuilder().setLimit(serializationFig.getMaxLoadSize()).build())
.execute().getResult().iterator();
+ if( !results.hasNext()){
+ if(logger.isTraceEnabled()){
+ logger.trace("No partitions returned for unique value lookup");
+ }
+ }
+
while ( results.hasNext() )
@@ -291,6 +297,10 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
//sanity check, nothing to do, skip it
if ( !columnList.hasNext() ) {
+ if(logger.isTraceEnabled()){
+ logger.trace("No cells exist in partition for unique value [{}={}]",
+ field.getName(), field.getValue().toString());
+ }
continue;
}
@@ -317,12 +327,24 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
// set the initial candidate and move on
if (candidates.size() == 0) {
candidates.add(uniqueValue);
+
+ if (logger.isTraceEnabled()) {
+ logger.trace("First entry for unique value [{}={}] found for application [{}], adding " +
+ "entry with entity id [{}] and entity version [{}] to the candidate list and continuing",
+ field.getName(), field.getValue().toString(), applicationId.getType(),
+ uniqueValue.getEntityId().getUuid(), uniqueValue.getEntityVersion());
+ }
+
continue;
}
if(!useReadRepair){
// take only the first
+ if (logger.isTraceEnabled()) {
+ logger.trace("Read repair not enabled for this request of unique value [{}={}], breaking out" +
+ " of cell loop", field.getName(), field.getValue().toString());
+ }
break;
} else {
[10/16] usergrid git commit: Set consistency level to ALL for now on
unique value serialization strategy as any cassandra inconsistencies will
break the usage of the unique value index.
Posted by sn...@apache.org.
Set consistency level to ALL for now on unique value serialization strategy as any cassandra inconsistencies will break the usage of the unique value index.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/02e1619c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/02e1619c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/02e1619c
Branch: refs/heads/usergrid-1268-akka-211
Commit: 02e1619cbce1ada18d25048558ccdfc2a3341542
Parents: e0a5d98
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Jul 3 12:32:25 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Jul 3 12:32:25 2016 -0700
----------------------------------------------------------------------
.../collection/impl/EntityCollectionManagerImpl.java | 4 +++-
.../collection/mvcc/stage/write/WriteUniqueVerify.java | 5 +++--
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/02e1619c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 658c68c..5bdcf1d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -333,8 +333,10 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final UUID startTime = UUIDGenerator.newTimeUUID();
//Get back set of unique values that correspond to collection of fields
+ //Purposely use CL ALL as consistency is extremely important here, regardless of performance
UniqueValueSet set =
- uniqueValueSerializationStrategy.load( applicationScope, type, fields1 , uniqueIndexRepair);
+ uniqueValueSerializationStrategy
+ .load( applicationScope, ConsistencyLevel.CL_ALL, type, fields1 , uniqueIndexRepair);
//Short circuit if we don't have any uniqueValues from the given fields.
if ( !set.iterator().hasNext() ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/02e1619c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 53160dc..01e6621 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -123,8 +123,9 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
// loading will retrieve the oldest unique value entry for the field
// don't use read repair on this pre-write check
- UniqueValueSet set = uniqueValueStrat.load(scope, written.getEntityId().getType(),
- Collections.singletonList(written.getField()), false);
+ // use CL ALL as consistency is extremely important here, more so than performance
+ UniqueValueSet set = uniqueValueStrat.load(scope, ConsistencyLevel.CL_ALL,
+ written.getEntityId().getType(), Collections.singletonList(written.getField()), false);
set.forEach(uniqueValue -> {
[03/16] usergrid git commit: Make logging more readable.
Posted by sn...@apache.org.
Make logging more readable.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/61aa0377
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/61aa0377
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/61aa0377
Branch: refs/heads/usergrid-1268-akka-211
Commit: 61aa03775eaca435d20d8a2e5af47d0dcb0f7641
Parents: 57231f0
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Jun 30 22:09:47 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Jun 30 22:09:47 2016 -0700
----------------------------------------------------------------------
.../impl/UniqueValueSerializationStrategyImpl.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61aa0377/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
index f37e219..d7b1be9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyImpl.java
@@ -394,11 +394,14 @@ public abstract class UniqueValueSerializationStrategyImpl<FieldKey, EntityKey>
}
// take the last candidate ( should be the latest version) and add to the result set
+
+ final UniqueValue returnValue = candidates.get(candidates.size() -1);
if(logger.isTraceEnabled()){
- logger.trace("Adding unique value [{}] to response set.",
- candidates.get(candidates.size() -1));
+ logger.trace("Adding unique value [{}={}] with entity id [{}] and entity version [{}] to response set",
+ returnValue.getField().getName(), returnValue.getField().getValue().toString(),
+ returnValue.getEntityId().getUuid(), returnValue.getEntityVersion());
}
- uniqueValueSet.addValue(candidates.get(candidates.size() -1));
+ uniqueValueSet.addValue(returnValue);
}
[15/16] usergrid git commit: Fixes to tests and test properties.
Posted by sn...@apache.org.
Fixes to tests and test properties.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/917f0e32
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/917f0e32
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/917f0e32
Branch: refs/heads/usergrid-1268-akka-211
Commit: 917f0e32ea72734b98902ef041a6fe5302495d4b
Parents: 996bf09
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 12:10:01 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 12:10:01 2016 -0400
----------------------------------------------------------------------
.../UniqueValuesServiceDeleteTest.java | 114 +++++++------------
1 file changed, 39 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/917f0e32/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
index 3f58834..397ec78 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceDeleteTest.java
@@ -22,6 +22,7 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.Multimaps;
import com.google.inject.Inject;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.collection.AbstractUniqueValueTest;
@@ -52,6 +53,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import static junit.framework.TestCase.fail;
+
/**
* Test the unique values service.
@@ -86,7 +89,7 @@ public class UniqueValuesServiceDeleteTest extends AbstractUniqueValueTest {
@Before
public void initAkka() {
// each test class needs unique port number
- initAkka( 2555, actorSystemManager, uniqueValuesService );
+ initAkka( 2559, actorSystemManager, uniqueValuesService );
}
@@ -94,92 +97,53 @@ public class UniqueValuesServiceDeleteTest extends AbstractUniqueValueTest {
* Use multiple threads to attempt to create entities with duplicate usernames.
*/
@Test
- public void testDuplicatePrevention() throws Exception {
+ public void testUniqueValueCleanup() throws Exception {
initAkka();
- final AtomicInteger successCounter = new AtomicInteger( 0 );
- final AtomicInteger errorCounter = new AtomicInteger( 0 );
-
- Multimap<String, Entity> usersCreated =
- generateDuplicateUsers( numUsers, successCounter, errorCounter );
-
- int userCount = 0;
- int usernamesWithDuplicates = 0;
- for ( String username : usersCreated.keySet() ) {
- Collection<Entity> users = usersCreated.get( username );
- if ( users.size() > 1 ) {
- usernamesWithDuplicates++;
- }
- userCount++;
- }
-
- Assert.assertEquals( 0, usernamesWithDuplicates );
-
- Assert.assertEquals( numUsers, successCounter.get() );
- Assert.assertEquals( 0, errorCounter.get() );
- Assert.assertEquals( numUsers, usersCreated.size() );
- Assert.assertEquals( numUsers, userCount );
- }
-
-
- private Multimap<String, Entity> generateDuplicateUsers(
- int numUsers, AtomicInteger successCounter, AtomicInteger errorCounter ) {
-
ApplicationScope context = new ApplicationScopeImpl( new SimpleId( "organization" ) );
EntityCollectionManager manager = factory.createCollectionManager( context );
- Multimap<String, Entity> usersCreated =
- Multimaps.synchronizedListMultimap( ArrayListMultimap.create() );
-
- ExecutorService execService = Executors.newFixedThreadPool( poolSize );
+ String username = RandomStringUtils.randomAlphanumeric( 20 );
- for (int i = 0; i < numUsers; i++) {
-
- // multiple threads simultaneously trying to create a user with the same propertyName
- for (int j = 0; j < numThreads; j++) {
- String username = "user_" + i;
-
- execService.submit( () -> {
-
- try {
-
- // give entity two unqiue fields username and email
- Entity newEntity = new Entity( new SimpleId( "user" ) );
- newEntity.setField( new StringField( "username", username, true ) );
- newEntity.setField( new StringField( "email", username + "@example.org", true ) );
-
- Observable<Entity> observable = manager.write( newEntity, null );
- Entity returned = observable.toBlocking().lastOrDefault( null );
-
- usersCreated.put( username, newEntity );
- successCounter.incrementAndGet();
-
- logger.debug("Created user {}", username);
-
- } catch ( Throwable t ) {
- if ( t instanceof WriteUniqueVerifyException) {
- // we expect lots of these
- } else {
- errorCounter.incrementAndGet();
- logger.error( "Error creating user " + username, t );
- }
- }
+ // create user
+ Entity originalUser = null;
+ {
+ Entity newEntity = new Entity( new SimpleId( "user" ) );
+ newEntity.setField( new StringField( "username", username, true ) );
+ newEntity.setField( new StringField( "email", username + "@example.org", true ) );
+ Observable<Entity> observable = manager.write( newEntity, null );
+ originalUser = observable.toBlocking().lastOrDefault( null );
+ }
- } );
- }
+ // cannot create another user with same name
+ {
+ Entity newEntity = new Entity( new SimpleId( "user" ) );
+ newEntity.setField( new StringField( "username", username, true ) );
+ newEntity.setField( new StringField( "email", username + "@example.org", true ) );
+ try {
+ Observable<Entity> observable = manager.write( newEntity, null );
+ Entity returned = observable.toBlocking().lastOrDefault( null );
+ fail("Should not have created dupliate user");
+ } catch ( WriteUniqueVerifyException expected ) {}
}
- execService.shutdown();
- try {
- while (!execService.awaitTermination( 60, TimeUnit.SECONDS )) {
- System.out.println( "Waiting..." );
+ // delete user
+ manager.mark( originalUser.getId(), null ).toBlocking().firstOrDefault( null );
+
+ // now we can create another user with same name
+ {
+ Entity newEntity = new Entity( new SimpleId( "user" ) );
+ newEntity.setField( new StringField( "username", username, true ) );
+ newEntity.setField( new StringField( "email", username + "@example.org", true ) );
+ try {
+ Observable<Entity> observable = manager.write( newEntity, null );
+ Entity returned = observable.toBlocking().lastOrDefault( null );
+ } catch ( WriteUniqueVerifyException unexpected ) {
+ logger.error("Error creating user", unexpected);
+ fail("Still cannot create new user after delete");
}
- } catch (InterruptedException e) {
- e.printStackTrace();
}
-
- return usersCreated;
}
}
[13/16] usergrid git commit: Add delays to the App Delete tests as UV
cleanup and take a couple of seconds.
Posted by sn...@apache.org.
Add delays to the App Delete tests as UV cleanup and take a couple of seconds.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/db6950b9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/db6950b9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/db6950b9
Branch: refs/heads/usergrid-1268-akka-211
Commit: db6950b9a476e0f1b77f1333d8852486232e4a9c
Parents: 670248d
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Jul 5 15:16:13 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Jul 5 15:16:13 2016 -0400
----------------------------------------------------------------------
...ApplicationAlreadyExistsExceptionMapper.java | 32 ++++++
.../rest/applications/ApplicationDeleteIT.java | 115 +++++++++++++++++--
2 files changed, 135 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/db6950b9/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ApplicationAlreadyExistsExceptionMapper.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ApplicationAlreadyExistsExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ApplicationAlreadyExistsExceptionMapper.java
new file mode 100644
index 0000000..2c90f5d
--- /dev/null
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ApplicationAlreadyExistsExceptionMapper.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.exceptions;
+
+
+import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
+
+import javax.ws.rs.core.Response;
+
+import static javax.ws.rs.core.Response.Status.CONFLICT;
+
+
+public class ApplicationAlreadyExistsExceptionMapper extends AbstractExceptionMapper<ApplicationAlreadyExistsException> {
+ @Override
+ public Response toResponse(ApplicationAlreadyExistsException e ) {
+ return toResponse( CONFLICT, e );
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/db6950b9/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
index 5858289..cc635ed 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationDeleteIT.java
@@ -51,6 +51,55 @@ public class ApplicationDeleteIT extends AbstractRestIT {
public static final int INDEXING_WAIT = 3000;
+ @Test
+ public void testBasicOperation() throws Exception {
+
+ // create app with a collection of "things"
+
+ String orgName = clientSetup.getOrganization().getName();
+ String appToDeleteName = clientSetup.getAppName() + "_appToDelete";
+ Token orgAdminToken = getAdminToken( clientSetup.getUsername(), clientSetup.getUsername());
+
+ List<Entity> entities = new ArrayList<>();
+
+ UUID appToDeleteId = createAppWithCollection(orgName, appToDeleteName, orgAdminToken, entities);
+
+ // delete the app
+
+ clientSetup.getRestClient().management().orgs()
+ .org(orgName).apps().app(appToDeleteId.toString() ).getTarget()
+ .queryParam("access_token", orgAdminToken.getAccessToken() )
+ .queryParam(CONFIRM_APPLICATION_IDENTIFIER, appToDeleteId)
+ .request().delete();
+
+ // test that we can create a new application with the same name
+
+ ApiResponse appCreateAgainResponse = null;
+
+ int retries = 0;
+ while ( retries++ < 100 ) {
+ try {
+ appCreateAgainResponse = clientSetup.getRestClient()
+ .management().orgs().org( orgName ).app().getTarget()
+ .queryParam( "access_token", orgAdminToken.getAccessToken() ).request()
+ .post( javax.ws.rs.client.Entity.json( new Application( appToDeleteName ) ), ApiResponse.class );
+
+ break;
+
+ } catch (Exception e) {
+ logger.error("App not deleted yet. Waiting ... ({})", retries);
+ Thread.sleep( 1000 );
+ }
+ }
+
+ Assert.assertNotNull( appCreateAgainResponse );
+
+ Assert.assertEquals("Must be able to create app with same name as deleted app",
+ (orgName + "/" + appToDeleteName).toLowerCase(),
+ appCreateAgainResponse.getEntities().get(0).get( "name" ));
+ }
+
+
/**
* Test most common use cases.
* <pre>
@@ -62,7 +111,7 @@ public class ApplicationDeleteIT extends AbstractRestIT {
* </pre>
*/
@Test
- public void testBasicOperation() throws Exception {
+ public void testCommonUseCases() throws Exception {
// create app with a collection of "things"
@@ -91,9 +140,11 @@ public class ApplicationDeleteIT extends AbstractRestIT {
.request()
.delete();
+
// test that we can no longer get the app
- try {
+ try { // using /management/orgs/{org-name}/app/{app-name}
+
clientSetup.getRestClient().management().orgs()
.org(orgName).apps().app(appToDeleteName).getTarget()
.queryParam("access_token", orgAdminToken.getAccessToken())
@@ -109,7 +160,8 @@ public class ApplicationDeleteIT extends AbstractRestIT {
}
- try {
+ try { // using /{org-name}/{app-name} path
+
clientSetup.getRestClient().org( orgName ).app( appToDeleteName ).getTarget()
.queryParam( "access_token", orgAdminToken.getAccessToken() ).request()
.get( ApiResponse.class );
@@ -185,11 +237,25 @@ public class ApplicationDeleteIT extends AbstractRestIT {
// test that we can create a new application with the same name
- ApiResponse appCreateAgainResponse = clientSetup.getRestClient()
- .management().orgs().org( orgName ).app().getTarget()
- .queryParam( "access_token", orgAdminToken.getAccessToken() )
- .request()
- .post( javax.ws.rs.client.Entity.json( new Application( appToDeleteName ) ), ApiResponse.class );
+ ApiResponse appCreateAgainResponse = null;
+
+ int retries = 0;
+ while ( retries++ < 20 ) {
+ try {
+ appCreateAgainResponse = clientSetup.getRestClient()
+ .management().orgs().org( orgName ).app().getTarget()
+ .queryParam( "access_token", orgAdminToken.getAccessToken() ).request()
+ .post( javax.ws.rs.client.Entity.json( new Application( appToDeleteName ) ), ApiResponse.class );
+
+ break;
+
+ } catch (Exception e) {
+ logger.error("App not deleted yet. Waiting ... ({})", retries);
+ Thread.sleep( 1000 );
+ }
+ }
+
+ Assert.assertNotNull( appCreateAgainResponse );
Assert.assertEquals("Must be able to create app with same name as deleted app",
(orgName + "/" + appToDeleteName).toLowerCase(),
@@ -312,7 +378,20 @@ public class ApplicationDeleteIT extends AbstractRestIT {
// create new app with same name
- createAppWithCollection(orgName, appToDeleteName, orgAdminToken, entities);
+ UUID newAppId = null;
+ int retries = 0;
+ while ( retries++ < 20 ) {
+ try {
+ newAppId = createAppWithCollection( orgName, appToDeleteName, orgAdminToken, entities );
+ break;
+
+ } catch ( Exception e ) {
+ logger.error("Application not deleted yet. Waiting... ({})", retries);
+ Thread.sleep(1000);
+ }
+ }
+
+ Assert.assertNotNull( newAppId );
// attempt to restore original app, should get 409
@@ -349,12 +428,24 @@ public class ApplicationDeleteIT extends AbstractRestIT {
.orgs().org( orgName ).apps().app( appToDeleteId.toString() ).getTarget()
.queryParam( "access_token", orgAdminToken.getAccessToken() )
.queryParam( CONFIRM_APPLICATION_IDENTIFIER, appToDeleteId )
- .request()
- .delete();
+ .request().delete();
// create new app with same name
- UUID newAppId = createAppWithCollection(orgName, appToDeleteName, orgAdminToken, entities);
+ UUID newAppId = null;
+ int retries = 0;
+ while ( retries++ < 20 ) {
+ try {
+ newAppId = createAppWithCollection( orgName, appToDeleteName, orgAdminToken, entities );
+ break;
+
+ } catch ( Exception e ) {
+ logger.error("Application not deleted yet. Waiting... ({})", retries);
+ Thread.sleep(1000);
+ }
+ }
+
+ Assert.assertNotNull( newAppId );
// attempt to delete new app, it should fail
[16/16] usergrid git commit: Fixing logic around entity delete to
ensure that UniqueValues and ReservationCache are both cleaned up.
Posted by sn...@apache.org.
Fixing logic around entity delete to ensure that UniqueValues and ReservationCache are both cleaned up.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e6c59ee3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e6c59ee3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e6c59ee3
Branch: refs/heads/usergrid-1268-akka-211
Commit: e6c59ee3fdc77ce32874ca6b2e0f5fc22a205a20
Parents: 917f0e3
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jul 7 12:12:42 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jul 7 12:12:42 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../resources/usergrid-custom-test.properties | 13 ++++
.../actorsystem/ActorSystemManagerImpl.java | 5 +-
.../exception/WriteUniqueVerifyException.java | 2 +-
.../mvcc/stage/delete/MarkCommit.java | 34 +++++++-
.../mvcc/stage/write/WriteUniqueVerify.java | 3 +-
.../uniquevalues/ReservationCache.java | 5 ++
.../uniquevalues/ReservationCacheActor.java | 13 +++-
.../uniquevalues/UniqueValueActor.java | 81 +++++++++++---------
.../uniquevalues/UniqueValuesService.java | 15 ++++
.../uniquevalues/UniqueValuesServiceImpl.java | 34 ++++++--
.../uniquevalues/UniqueValuesTable.java | 16 ++--
.../uniquevalues/UniqueValuesTableImpl.java | 8 ++
13 files changed, 170 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 7006f44..41613ac 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2863,7 +2863,7 @@ public class CpEntityManager implements EntityManager {
throws DuplicateUniquePropertyExistsException {
// we may have multiple conflicts, but caller expects only one
- Map<String, Field> violiations = wuve.getVioliations();
+ Map<String, Field> violiations = wuve.getViolations();
if ( violiations != null ) {
Field conflict = violiations.get( violiations.keySet().iterator().next() );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index 0fa9573..dd6612f 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -34,3 +34,16 @@ elasticsearch.buffer_timeout=1
# This property is required to be set and cannot be defaulted anywhere
usergrid.cluster_name=usergrid
+# --- Start: Usergrid cluster/actor system settings
+
+usergrid.cluster.enabled=true
+usergrid.cluster.hostname=localhost
+usergrid.cluster.region.local=us-east
+usergrid.cluster.region.list=us-east
+usergrid.cluster.seeds=us-east\:localhost
+
+collection.uniquevalues.actors=300
+collection.uniquevalues.authoritative.region=us-east
+
+# --- End: Usergrid cluster/actor system settings
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index 099b875..8399979 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -282,12 +282,13 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
String region = currentRegion;
List<String> seeds = getSeedsByRegion().get( region );
- int lastColon = seeds.get(0).lastIndexOf(":") + 1;
- final Integer regionPort = Integer.parseInt( seeds.get(0).substring( lastColon ));
logger.info( "Akka Config for region {} is:\n" + " Hostname {}\n" + " Seeds {}\n",
region, hostname, seeds );
+ int lastColon = seeds.get(0).lastIndexOf(":") + 1;
+ final Integer regionPort = Integer.parseInt( seeds.get(0).substring( lastColon ));
+
Map<String, Object> configMap = new HashMap<String, Object>() {{
put( "akka", new HashMap<String, Object>() {{
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
index 81165fa..a223a9e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
@@ -40,7 +40,7 @@ public class WriteUniqueVerifyException extends CollectionRuntimeException {
/**
* Get map of Fields in violation, keyed by field name.
*/
- public Map<String, Field> getVioliations() {
+ public Map<String, Field> getViolations() {
return violations;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index 460c271..70c785e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@ -21,6 +21,10 @@ package org.apache.usergrid.persistence.collection.mvcc.stage.delete;
import java.util.UUID;
+import org.apache.usergrid.persistence.actorsystem.ActorSystemFig;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueException;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesFig;
+import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValuesService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,12 +64,19 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
private final SerializationFig serializationFig;
private final UniqueValueSerializationStrategy uniqueValueStrat;
private final Keyspace keyspace;
+ private final ActorSystemFig actorSystemFig;
+ private final UniqueValuesFig uniqueValuesFig;
+ private final UniqueValuesService uniqueValuesService;
@Inject
public MarkCommit( final MvccLogEntrySerializationStrategy logStrat,
final MvccEntitySerializationStrategy entityStrat,
- final UniqueValueSerializationStrategy uniqueValueStrat, final SerializationFig serializationFig,
+ final UniqueValueSerializationStrategy uniqueValueStrat,
+ final SerializationFig serializationFig,
+ final ActorSystemFig actorSystemFig,
+ final UniqueValuesFig uniqueValuesFig,
+ final UniqueValuesService uniqueValuesService,
final Keyspace keyspace ) {
@@ -76,6 +87,9 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
this.entityStrat = entityStrat;
this.serializationFig = serializationFig;
this.uniqueValueStrat = uniqueValueStrat;
+ this.actorSystemFig = actorSystemFig;
+ this.uniqueValuesFig = uniqueValuesFig;
+ this.uniqueValuesService = uniqueValuesService;
this.keyspace = keyspace;
}
@@ -111,6 +125,24 @@ public class MarkCommit implements Action1<CollectionIoEvent<MvccEntity>> {
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to mark entry as deleted" );
}
+
+ // actorSystemFig may be null in testing
+ if ( actorSystemFig != null && actorSystemFig.getEnabled() ) {
+
+ String region = idIoEvent.getRegion();
+ if ( region == null ) {
+ region = uniqueValuesFig.getAuthoritativeRegion();
+ }
+ if ( region == null ) {
+ region = actorSystemFig.getRegionLocal();
+ }
+
+ try {
+ uniqueValuesService.releaseUniqueValues( applicationScope, entityId, version, region );
+ } catch (UniqueValueException e) {
+ throw new RuntimeException( "Unable to release unique values for entity " + entityId );
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index e7ba967..1f86440 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -219,8 +219,7 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
logger.trace("Pre-write unique violations found, raising exception before executing first write");
}
- throw new WriteUniqueVerifyException(mvccEntity, scope,
- preWriteUniquenessViolations );
+ throw new WriteUniqueVerifyException(mvccEntity, scope, preWriteUniquenessViolations );
}
//short circuit nothing to do
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java
index f1e68b2..8e2e51e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCache.java
@@ -72,6 +72,11 @@ public class ReservationCache {
cache.invalidate( cancellation.getConsistentHashKey() );
}
+ public void cancelReservation( UniqueValueActor.Response response ) {
+ if ( ttl == 0 ) { return; }
+ cache.invalidate( response.getConsistentHashKey() );
+ }
+
public CacheStats getStats() {
return cache.stats();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
index 51f5c8c..158b099 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/ReservationCacheActor.java
@@ -54,12 +54,19 @@ public class ReservationCacheActor extends UntypedActor {
}
} else if ( msg instanceof UniqueValueActor.Cancellation ) {
- UniqueValueActor.Cancellation can = (UniqueValueActor.Cancellation)msg;
+ UniqueValueActor.Cancellation can = (UniqueValueActor.Cancellation) msg;
ReservationCache.getInstance().cancelReservation( can );
- if ( ++cancellationCount % 10 == 0 ) {
- logger.info("Received {} cancellations", cancellationCount);
+ if (++cancellationCount % 10 == 0) {
+ logger.info( "Received {} cancellations", cancellationCount );
}
+ logger.debug("Removing cancelled {} from reservation cache", can.getConsistentHashKey());
+
+ } else if ( msg instanceof UniqueValueActor.Response ) {
+ UniqueValueActor.Response response = (UniqueValueActor.Response) msg;
+ ReservationCache.getInstance().cancelReservation( response );
+
+ logger.info("Removing completed {} from reservation cache", response.getConsistentHashKey());
} else if (msg instanceof DistributedPubSubMediator.SubscribeAck) {
logger.debug( "subscribing" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
index a14c63e..501037f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValueActor.java
@@ -16,10 +16,7 @@
*/
package org.apache.usergrid.persistence.collection.uniquevalues;
-import akka.actor.ActorRef;
import akka.actor.UntypedActor;
-import akka.cluster.pubsub.DistributedPubSub;
-import akka.cluster.pubsub.DistributedPubSubMediator;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -51,8 +48,6 @@ public class UniqueValueActor extends UntypedActor {
// TODO: is there a way to avoid this ugly kludge? see also: ClusterSingletonRouter
this.table = UniqueValuesServiceImpl.injector.getInstance( UniqueValuesTable.class );
this.actorSystemManager = UniqueValuesServiceImpl.injector.getInstance( ActorSystemManager.class );
-
- //logger.info("UniqueValueActor {} is live with table {}", name, table);
}
@Override
@@ -77,24 +72,27 @@ public class UniqueValueActor extends UntypedActor {
if ( owner != null && owner.equals( res.getOwner() )) {
// sender already owns this unique value
- getSender().tell( new Response( Response.Status.IS_UNIQUE ), getSender() );
+ getSender().tell( new Response( Response.Status.IS_UNIQUE, res.getConsistentHashKey() ),
+ getSender() );
return;
} else if ( owner != null && !owner.equals( res.getOwner() )) {
// tell sender value is not unique
- getSender().tell( new Response( Response.Status.NOT_UNIQUE ), getSender() );
+ getSender().tell( new Response( Response.Status.NOT_UNIQUE, res.getConsistentHashKey() ),
+ getSender() );
return;
}
table.reserve( res.getApplicationScope(), res.getOwner(), res.getOwnerVersion(), res.getField() );
- getSender().tell( new Response( Response.Status.IS_UNIQUE ), getSender() );
+ getSender().tell( new Response( Response.Status.IS_UNIQUE, res.getConsistentHashKey() ),
+ getSender() );
actorSystemManager.publishToAllRegions( "content", new Reservation( res ), getSelf() );
} catch (Throwable t) {
- getSender().tell( new Response( Response.Status.ERROR ), getSender() );
+ getSender().tell( new Response( Response.Status.ERROR, res.getConsistentHashKey() ), getSender() );
logger.error( "Error processing request", t );
@@ -112,23 +110,29 @@ public class UniqueValueActor extends UntypedActor {
if ( owner != null && !owner.equals( con.getOwner() )) {
// cannot reserve, somebody else owns the unique value
- getSender().tell( new Response( Response.Status.NOT_UNIQUE ), getSender() );
+ Response response = new Response( Response.Status.NOT_UNIQUE, con.getConsistentHashKey());
+ getSender().tell( response, getSender() );
+ actorSystemManager.publishToAllRegions( "content", response, getSelf() );
return;
} else if ( owner == null ) {
// cannot commit without first reserving
- getSender().tell( new Response( Response.Status.BAD_REQUEST ), getSender() );
+ Response response = new Response( Response.Status.BAD_REQUEST, con.getConsistentHashKey());
+ getSender().tell( response, getSender() );
+ actorSystemManager.publishToAllRegions( "content", response, getSelf() );
return;
}
table.confirm( con.getApplicationScope(), con.getOwner(), con.getOwnerVersion(), con.getField() );
- getSender().tell( new Response( Response.Status.IS_UNIQUE ), getSender() );
+ Response response = new Response( Response.Status.IS_UNIQUE, con.getConsistentHashKey() );
+ getSender().tell( response, getSender() );
- actorSystemManager.publishToAllRegions( "content", new Reservation( con ), getSelf() );
+ actorSystemManager.publishToAllRegions( "content", response, getSelf() );
} catch (Throwable t) {
- getSender().tell( new Response( Response.Status.ERROR ), getSender() );
+ getSender().tell( new Response( Response.Status.ERROR, con.getConsistentHashKey() ),
+ getSender() );
logger.error( "Error processing request", t );
} finally {
@@ -144,23 +148,34 @@ public class UniqueValueActor extends UntypedActor {
if ( owner != null && !owner.equals( can.getOwner() )) {
// cannot cancel, somebody else owns the unique value
- getSender().tell( new Response( Response.Status.NOT_UNIQUE ), getSender() );
+ getSender().tell( new Response( Response.Status.NOT_UNIQUE, can.getConsistentHashKey() ),
+ getSender() );
return;
} else if ( owner == null ) {
+
// cannot cancel unique value that does not exist
- getSender().tell( new Response( Response.Status.BAD_REQUEST ), getSender() );
+ getSender().tell( new Response( Response.Status.BAD_REQUEST, can.getConsistentHashKey() ),
+ getSender() );
+
+ // unique value record may have already been cleaned up, also clear cache
+ actorSystemManager.publishToAllRegions( "content", new Cancellation( can ), getSelf() );
+
return;
}
- table.confirm( can.getApplicationScope(), can.getOwner(), can.getOwnerVersion(), can.getField() );
+ table.cancel( can.getApplicationScope(), can.getOwner(), can.getOwnerVersion(), can.getField() );
+
+ logger.debug("Removing {} from unique values table", can.getConsistentHashKey());
- getSender().tell( new Response( Response.Status.SUCCESS ), getSender() );
+ getSender().tell( new Response( Response.Status.SUCCESS, can.getConsistentHashKey() ),
+ getSender() );
- actorSystemManager.publishToAllRegions( "content", new Reservation( can ), getSelf() );
+ actorSystemManager.publishToAllRegions( "content", new Cancellation( can ), getSelf() );
} catch (Throwable t) {
- getSender().tell( new Response( Response.Status.ERROR ), getSender() );
+ getSender().tell( new Response( Response.Status.ERROR, can.getConsistentHashKey() ),
+ getSender() );
logger.error( "Error processing request", t );
}
@@ -188,7 +203,7 @@ public class UniqueValueActor extends UntypedActor {
this.field = field;
StringBuilder sb = new StringBuilder();
- sb.append( applicationScope.getApplication() );
+ sb.append( applicationScope.getApplication().getUuid() );
sb.append(":");
sb.append( owner.getType() );
sb.append(":");
@@ -198,22 +213,7 @@ public class UniqueValueActor extends UntypedActor {
this.consistentHashKey = sb.toString();
}
public Request( Request req ) {
-
- this.applicationScope = req.applicationScope;
- this.owner = req.owner;
- this.ownerVersion = req.ownerVersion;
- this.field = req.field;
-
- StringBuilder sb = new StringBuilder();
- sb.append( req.applicationScope.getApplication() );
- sb.append(":");
- sb.append( req.owner.getType() );
- sb.append(":");
- sb.append( req.field.getName() );
- sb.append(":");
- sb.append( req.field.getValue().toString() );
- this.consistentHashKey = sb.toString();
-
+ this( req.getApplicationScope(), req.getOwner(), req.getOwnerVersion(), req.getField() );
}
public ApplicationScope getApplicationScope() {
return applicationScope;
@@ -238,13 +238,18 @@ public class UniqueValueActor extends UntypedActor {
public static class Response implements Serializable {
public enum Status { IS_UNIQUE, NOT_UNIQUE, SUCCESS, ERROR, BAD_REQUEST }
final Status status;
+ final String consistentHashKey;
- public Response(Status status) {
+ public Response(Status status, String consistentHashKey ) {
this.status = status;
+ this.consistentHashKey = consistentHashKey;
}
public Status getStatus() {
return status;
}
+ public String getConsistentHashKey() {
+ return consistentHashKey;
+ }
}
public static class Reservation extends Request implements Serializable {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
index e9a9f50..5deddd6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesService.java
@@ -20,8 +20,11 @@ package org.apache.usergrid.persistence.collection.uniquevalues;
import org.apache.usergrid.persistence.actorsystem.RouterProducer;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.field.Field;
import java.util.UUID;
@@ -55,4 +58,16 @@ public interface UniqueValuesService extends RouterProducer {
*/
void confirmUniqueValues( ApplicationScope scope, Entity entity, UUID version , String region )
throws UniqueValueException;
+
+ /**
+ * Release unique values held by an entity.
+ *
+ * @param scope Application scope of entity.
+ * @param entityId Id of Entity with unique values to be released
+ * @param version Version of entity.
+ * @param region Authoritative Region to be used for this entity or null to use current region.
+ * @throws UniqueValueException if unique values cannot be reserved.
+ */
+ void releaseUniqueValues( ApplicationScope scope, Id entityId, UUID version, String region )
+ throws UniqueValueException;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
index fcee8fa..4102090 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesServiceImpl.java
@@ -35,8 +35,10 @@ import com.google.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.persistence.actorsystem.ActorSystemManager;
import org.apache.usergrid.persistence.actorsystem.GuiceActorProducer;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,6 +46,7 @@ import scala.concurrent.Await;
import scala.concurrent.Future;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@@ -109,13 +112,12 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
try {
cancelUniqueField( scope, entity, version, field, region );
} catch (Throwable ignored) {
- logger.debug( "Error canceling unique field", ignored );
+ logger.error( "Error canceling unique field", ignored );
}
}
}
throw e;
}
-
}
@@ -149,6 +151,21 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
}
+ @Override
+ public void releaseUniqueValues(ApplicationScope scope, Id entityId, UUID version, String region)
+ throws UniqueValueException {
+
+ ready();
+
+ Iterator<UniqueValue> iterator = table.getUniqueValues( scope, entityId );
+
+ while ( iterator.hasNext() ) {
+ UniqueValue uniqueValue = iterator.next();
+ cancelUniqueField( scope, entityId, uniqueValue.getEntityVersion(), uniqueValue.getField(), region );
+ }
+ }
+
+
private void reserveUniqueField(
ApplicationScope scope, Entity entity, UUID version, Field field, String region ) throws UniqueValueException {
@@ -177,11 +194,18 @@ public class UniqueValuesServiceImpl implements UniqueValuesService {
}
- private void cancelUniqueField(
- ApplicationScope scope, Entity entity, UUID version, Field field, String region ) throws UniqueValueException {
+ private void cancelUniqueField( ApplicationScope scope,
+ Entity entity, UUID version, Field field, String region ) throws UniqueValueException {
+
+ cancelUniqueField( scope, entity.getId(), version, field, region );
+ }
+
+
+ private void cancelUniqueField( ApplicationScope scope,
+ Id entityId, UUID version, Field field, String region ) throws UniqueValueException {
UniqueValueActor.Cancellation request = new UniqueValueActor.Cancellation(
- scope, entity.getId(), version, field );
+ scope, entityId, version, field );
if ( actorSystemManager.getCurrentRegion().equals( region ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java
index 0e69ef7..1bb2534 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTable.java
@@ -19,24 +19,24 @@
package org.apache.usergrid.persistence.collection.uniquevalues;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.field.Field;
+import java.util.Iterator;
import java.util.UUID;
public interface UniqueValuesTable {
- Id lookupOwner(
- ApplicationScope applicationScope, String type, Field field ) throws ConnectionException;
+ Id lookupOwner( ApplicationScope scope, String type, Field field ) throws ConnectionException;
- void reserve(
- ApplicationScope applicationScope, Id owner, UUID version, Field field ) throws ConnectionException;
+ void reserve( ApplicationScope scope, Id owner, UUID version, Field field ) throws ConnectionException;
- void confirm(
- ApplicationScope applicationScope, Id owner, UUID version, Field field ) throws ConnectionException;
+ void confirm( ApplicationScope scope, Id owner, UUID version, Field field ) throws ConnectionException;
- void cancel(
- ApplicationScope applicationScope, Id owner, UUID version, Field field ) throws ConnectionException;
+ void cancel( ApplicationScope scope, Id owner, UUID version, Field field ) throws ConnectionException;
+
+ Iterator<UniqueValue> getUniqueValues(ApplicationScope scope, Id entityId );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e6c59ee3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java
index 9cb13be..2cad32c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/uniquevalues/UniqueValuesTableImpl.java
@@ -33,7 +33,9 @@ import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
import java.util.Collections;
+import java.util.Iterator;
import java.util.UUID;
@@ -83,4 +85,10 @@ public class UniqueValuesTableImpl implements UniqueValuesTable {
final MutationBatch write = strat.delete( scope, uv );
write.execute();
}
+
+ @Override
+ public Iterator<UniqueValue> getUniqueValues(ApplicationScope scope, Id entityId) {
+ return strat.getAllUniqueFields( scope, entityId );
+ }
+
}
[05/16] usergrid git commit: add missing license headers
Posted by sn...@apache.org.
add missing license headers
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1b4e1172
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1b4e1172
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1b4e1172
Branch: refs/heads/usergrid-1268-akka-211
Commit: 1b4e1172b516eed4746b7a4d7999a329b87b6796
Parents: f0c9fd4
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jul 1 15:17:06 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jul 1 15:17:06 2016 -0400
----------------------------------------------------------------------
.../src/test/resources/usergrid-CHOP.properties | 18 ++++++++++++++++++
.../src/test/resources/usergrid-UNIT.properties | 19 +++++++++++++++++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b4e1172/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
index 78e3400..d404b1e 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
@@ -1,3 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
# These are for CHOP environment settings
cassandra.connections=20
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b4e1172/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
index 44dc399..81a4c17 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
@@ -1,3 +1,22 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
# Keep nothing but overriding test defaults in here
cassandra.connections=50
cassandra.port=9160