You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/08/14 18:41:38 UTC
[07/11] git commit: Fix for problem in unique values implementation
in Core Persistence, and tests to verify new fix.
Fix for problem in unique values implementation in Core Persistence, and tests to verify new fix.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ab8ea15d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ab8ea15d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ab8ea15d
Branch: refs/heads/two-dot-o
Commit: ab8ea15d35845b86e90d313249238dd795830cd0
Parents: 053b027
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Aug 13 17:03:17 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Aug 13 17:03:17 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 3 +-
.../corepersistence/CpEntityMapUtils.java | 24 ++++---
.../corepersistence/TestGuiceModule.java | 27 --------
.../mvcc/stage/write/WriteCommit.java | 8 ++-
.../mvcc/stage/write/WriteUniqueVerify.java | 2 +-
.../serialization/SerializationFig.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 9 +++
.../src/test/resources/dynamic-test.properties | 4 +-
.../src/test/resources/usergrid-UNIT.properties | 3 +-
.../collection/DuplicateNameIT.java | 72 ++++++++++++++++++++
.../resources/usergrid-custom-test.properties | 14 ++--
11 files changed, 119 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 906b4f4..f02a6a1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2523,7 +2523,8 @@ public class CpEntityManager implements EntityManager {
return entity;
}
- org.apache.usergrid.persistence.model.entity.Entity cpEntity = entityToCpEntity( entity, importId );
+ org.apache.usergrid.persistence.model.entity.Entity cpEntity =
+ entityToCpEntity( entity, importId );
// prepare to write and index Core Persistence Entity into default scope
CollectionScope collectionScope = new CollectionScopeImpl(
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityMapUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityMapUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityMapUtils.java
index ecf4f2c..560f283 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityMapUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityMapUtils.java
@@ -51,12 +51,16 @@ import org.apache.usergrid.persistence.model.field.value.Location;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Utilities for converting entities to/from maps suitable for Core Persistence.
* Aware of unique properties via Schema.
*/
class CpEntityMapUtils {
+ private static final Logger logger = LoggerFactory.getLogger( CpEntityMapUtils.class );
public static ObjectMapper objectMapper = new ObjectMapper( );
@@ -73,31 +77,35 @@ class CpEntityMapUtils {
for ( String fieldName : map.keySet() ) {
Object value = map.get( fieldName );
- boolean unqiue = Schema.getDefaultSchema().isPropertyUnique(entityType, fieldName);
+ boolean unique = Schema.getDefaultSchema().isPropertyUnique(entityType, fieldName);
+
+// if ( unique ) {
+// logger.debug("{} is a unique property", fieldName );
+// }
if ( value instanceof String ) {
- entity.setField( new StringField( fieldName, (String)value, unqiue && topLevel ));
+ entity.setField( new StringField( fieldName, (String)value, unique && topLevel ));
} else if ( value instanceof Boolean ) {
- entity.setField( new BooleanField( fieldName, (Boolean)value, unqiue && topLevel ));
+ entity.setField( new BooleanField( fieldName, (Boolean)value, unique && topLevel ));
} else if ( value instanceof Integer ) {
- entity.setField( new IntegerField( fieldName, (Integer)value, unqiue && topLevel ));
+ entity.setField( new IntegerField( fieldName, (Integer)value, unique && topLevel ));
} else if ( value instanceof Double ) {
- entity.setField( new DoubleField( fieldName, (Double)value, unqiue && topLevel ));
+ entity.setField( new DoubleField( fieldName, (Double)value, unique && topLevel ));
} else if ( value instanceof Float ) {
- entity.setField( new FloatField( fieldName, (Float)value, unqiue && topLevel ));
+ entity.setField( new FloatField( fieldName, (Float)value, unique && topLevel ));
} else if ( value instanceof Long ) {
- entity.setField( new LongField( fieldName, (Long)value, unqiue && topLevel ));
+ entity.setField( new LongField( fieldName, (Long)value, unique && topLevel ));
} else if ( value instanceof List) {
entity.setField( listToListField( fieldName, (List)value, entityType ));
} else if ( value instanceof UUID) {
- entity.setField( new UUIDField( fieldName, (UUID)value, unqiue && topLevel ));
+ entity.setField( new UUIDField( fieldName, (UUID)value, unique && topLevel ));
} else if ( value instanceof Map ) {
processMapValue( value, fieldName, entity, entityType);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
index 863a043..6fce1dd 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/TestGuiceModule.java
@@ -19,37 +19,10 @@
package org.apache.usergrid.corepersistence;
import com.google.inject.AbstractModule;
-import com.netflix.config.ConfigurationManager;
-import java.io.IOException;
-import java.util.Properties;
public class TestGuiceModule extends AbstractModule {
- static {
-
- //--------------------------------------------------------------------
- // Bootstrap the config for Archaius Configuration Settings.
- // We don't want to bootstrap more than once per JVM
- //--------------------------------------------------------------------
-
- try {
- ConfigurationManager.loadCascadedPropertiesFromResources( "corepersistence" );
-
- Properties testProps = new Properties() {{
- String port = System.getProperty("cassandra.rpc_port");
- if ( port == null && "null".equals(port) ) {
- port = "9160";
- }
- put("cassandra.hosts", "localhost:" + port);
- }};
- ConfigurationManager.loadProperties( testProps );
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Cannot do much without properly loading our configuration.", e );
- }
- }
-
@Override
protected void configure() {
install( new GuiceModule() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
index 7147c8e..0004ddb 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
@@ -107,11 +107,15 @@ public class WriteCommit implements Func1<CollectionIoEvent<MvccEntity>, Entity>
// re-write the unique values but this time with no TTL
for ( Field field : mvccEntity.getEntity().get().getFields() ) {
+
if ( field.isUnique() ) {
- UniqueValue written = new UniqueValueImpl(
- ioEvent.getEntityCollection(), field, mvccEntity.getId(), mvccEntity.getVersion());
+
+ UniqueValue written = new UniqueValueImpl( ioEvent.getEntityCollection(), field,
+ mvccEntity.getEntity().get().getId(), mvccEntity.getEntity().get().getVersion());
MutationBatch mb = uniqueValueStrat.write( written );
+ LOG.debug("Finalizing {} unqiue value {}", field.getName(), field.getValue().toString());
+
// merge into our existing mutation batch
logMutation.mergeShallow( mb );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 830990e..abec0c9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -110,7 +110,7 @@ public class WriteUniqueVerify implements
// use write-first then read strategy
UniqueValue written = new UniqueValueImpl(
- ioevent.getEntityCollection(), field, entity.getId(), entity.getVersion() );
+ ioevent.getEntityCollection(), field, entity.getId(), mvccEntity.getVersion() );
// use TTL in case something goes wrong before entity is finally committed
MutationBatch mb = uniqueValueStrat.write( written, serializationFig.getTimeout() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 4094704..81302a6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -18,7 +18,7 @@ public interface SerializationFig extends GuicyFig {
* @return Timeout in seconds.
*/
@Key( "collection.stage.transient.timeout" )
- @Default( "60" )
+ @Default( "5" )
int getTimeout();
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
index f1d361c..04f55a7 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyIT.java
@@ -27,6 +27,7 @@ import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.stage.TestEntityGenerator;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -48,6 +49,9 @@ import org.junit.runner.RunWith;
@UseModules( TestCollectionModule.class )
public class WriteUniqueVerifyIT {
+ @Inject
+ SerializationFig serializationFig;
+
@Inject
@Rule
public MigrationManagerRule migrationManagerRule;
@@ -73,6 +77,11 @@ public class WriteUniqueVerifyIT {
Entity entityFetched = entityManager.load( entity.getId() ).toBlocking().last();
entityFetched.setField( new StringField("foo", "bar"));
+ // wait for temporary unique value records to time out
+ try {
+ Thread.sleep(serializationFig.getTimeout() * 1100);
+ } catch (InterruptedException ignored) { }
+
// another enity that tries to use two unique values already taken by first
final Entity entity2 = TestEntityGenerator.generateEntity();
entity2.setField(new StringField("name", "Aston Martin Vanquish", true));
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
index e656ddd..1e98f3c 100644
--- a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
+++ b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
@@ -7,6 +7,4 @@ cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
collections.keyspace=Usergrid_Collections
cassandra.timeout=5000
-
-
-
+collection.stage.transient.timeout=5
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
index 9785b25..4632027 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
@@ -10,4 +10,5 @@ cassandra.timeout=5000
collections.keyspace.strategy.options=replication_factor:1
collections.keyspace.strategy.class=SimpleStrategy
-collection.stage.transient.timeout=60
+
+collection.stage.transient.timeout=5
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java
new file mode 100644
index 0000000..0273f04
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/DuplicateNameIT.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.rest.applications.collection;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import java.io.IOException;
+import java.util.Map;
+import org.apache.usergrid.corepersistence.TestGuiceModule;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.rest.AbstractRestIT;
+import org.apache.usergrid.rest.TestContextSetup;
+import org.apache.usergrid.rest.test.resource.CustomCollection;
+import org.apache.usergrid.utils.MapUtils;
+import static org.junit.Assert.fail;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class DuplicateNameIT extends AbstractRestIT {
+
+ private static final Logger logger = LoggerFactory.getLogger( DuplicateNameIT.class );
+
+ @Rule
+ public TestContextSetup context = new TestContextSetup( this );
+
+ @Test
+ public void duplicateNamePrevention() {
+
+ CustomCollection things = context.application().collection( "things" );
+
+ Map<String, String> entity = MapUtils.hashMap( "name", "enzo" );
+
+ try {
+ things.create( entity );
+ } catch (IOException ex) {
+ logger.error("Cannot create entity", ex);
+ }
+
+ refreshIndex( context.getAppUuid() );
+
+ Injector injector = Guice.createInjector( new TestGuiceModule() );
+ SerializationFig sfig = injector.getInstance( SerializationFig.class );
+
+ // wait for any temporary unique value records to timeout
+ try { Thread.sleep( sfig.getTimeout() * 1100 ); } catch (InterruptedException ignored) {}
+
+ try {
+ things.create( entity );
+ fail("Should not have created duplicate entity");
+
+ } catch (Exception ex) {
+ // good
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ab8ea15d/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index b234fd6..fcb1630 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -2,15 +2,19 @@
usergrid.test=true
usergrid.mongo.disable=true
-
usergrid.counter.batch.size=1
-
swagger.basepath=http://sometestvalue
-elasticsearch.force_refresh=false
-
tomcat.startup=embedded
elasticsearch.startup=forked
+elasticsearch.force_refresh=false
+
+cassandra.startup=forked
+cassandra.connections=600
+cassandra.timeout=5000
+
+hystrix.threadpool.graph_user.coreSize=20
+hystrix.threadpool.graph_async.coreSize=20
-cassandra.startup=forked
\ No newline at end of file
+collection.stage.transient.timeout=5