You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/03/18 21:56:43 UTC
[43/50] incubator-usergrid git commit: Fixes bug in writeunique
verify failover command
Fixes bug in writeunique verify failover command
Fixes NoAWSCredsRule to use Assumptions to mark as ignored
Adds AWS creds rule to NotificationsServiceTest
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/03341191
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/03341191
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/03341191
Branch: refs/heads/USERGRID-460
Commit: 03341191a2109b8d297387c37b88df7af7b50218
Parents: f20dfd3
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 15 15:34:45 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 15 15:34:45 2015 -0600
----------------------------------------------------------------------
.../corepersistence/StaleIndexCleanupTest.java | 3 ++-
.../impl/EntityVersionCleanupTask.java | 7 +++--
.../mvcc/stage/write/WriteUniqueVerify.java | 24 +++++++++--------
.../collection/util/EntityUtils.java | 28 +++++++++++++++++---
.../index/impl/EsIndexBufferConsumerImpl.java | 14 ++++++++--
.../persistence/queue/NoAWSCredsRule.java | 17 ++++++++++++
stack/services/pom.xml | 13 ++++++++-
.../notifications/NotifiersServiceIT.java | 6 +++++
8 files changed, 90 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index ac04dcc..4bde50e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -392,9 +392,10 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
// wait for indexes to be cleared for the deleted entities
count = 0;
do {
+ queryCollectionEm("dogs", "select *");
Thread.sleep(100);
crs = queryCollectionCp("dogs", "dog", "select *");
- } while ( crs.size() == numEntities && count++ < 15 );
+ } while ( crs.size() != numEntities && count++ < 15 );
Assert.assertEquals("Expect candidates without earlier stale entities", crs.size(), numEntities);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 03a5ff6..65506ec 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -28,6 +28,7 @@ import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
@@ -158,10 +159,8 @@ public class EntityVersionCleanupTask implements Task<Void> {
final Entity entity = mvccEntity.getEntity().get();
//remove all unique fields from the index
- for ( final Field field : entity.getFields() ) {
- if ( !field.isUnique() ) {
- continue;
- }
+ for ( final Field field : EntityUtils.getUniqueFields(entity )) {
+
final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
final MutationBatch deleteMutation =
uniqueValueSerializationStrategy.delete( scope, unique );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 173efa7..5bdf3b9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -31,6 +31,8 @@ import com.netflix.hystrix.HystrixCommand;
import com.netflix.hystrix.HystrixCommandGroupKey;
import com.netflix.hystrix.HystrixThreadPoolProperties;
import com.netflix.hystrix.strategy.concurrency.HystrixRequestVariableLifecycle;
+
+import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
import org.slf4j.Logger;
@@ -108,20 +110,21 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
//
// Construct all the functions for verifying we're unique
//
- for ( final Field field : entity.getFields() ) {
+
+
+ for ( final Field field : EntityUtils.getUniqueFields(entity)) {
// if it's unique, create a function to validate it and add it to the list of
// concurrent validations
- if ( field.isUnique() ) {
- // use write-first then read strategy
- final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
- // use TTL in case something goes wrong before entity is finally committed
- final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
+ // use write-first then read strategy
+ final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() );
- batch.mergeShallow( mb );
- uniqueFields.add(field);
- }
+ // use TTL in case something goes wrong before entity is finally committed
+ final MutationBatch mb = uniqueValueStrat.write( scope, written, serializationFig.getTimeout() );
+
+ batch.mergeShallow( mb );
+ uniqueFields.add(field);
}
//short circuit nothing to do
@@ -175,12 +178,11 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
}
public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){
- Collection<Field> entityFields = entity.getFields();
//allocate our max size, worst case
//now get the set of fields back
final UniqueValueSet uniqueValues;
try {
- uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, entityFields );
+ uniqueValues = uniqueValueSerializationStrategy.load( scope,consistencyLevel, uniqueFields );
}
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to read from cassandra", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
index 6233075..cf964a3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/EntityUtils.java
@@ -1,8 +1,11 @@
package org.apache.usergrid.persistence.collection.util;
-import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.persistence.model.field.Field;
import org.apache.commons.lang3.reflect.FieldUtils;
@@ -16,9 +19,9 @@ import org.apache.usergrid.persistence.model.entity.Id;
public class EntityUtils {
- private static final Field VERSION = FieldUtils.getField( Entity.class, "version", true );
+ private static final java.lang.reflect.Field VERSION = FieldUtils.getField( Entity.class, "version", true );
- private static final Field ID = FieldUtils.getField( Entity.class, "id", true );
+ private static final java.lang.reflect.Field ID = FieldUtils.getField( Entity.class, "id", true );
/**
@@ -46,4 +49,23 @@ public class EntityUtils {
throw new RuntimeException( "Unable to set the field " + ID + " into the entity", e );
}
}
+
+
+ /**
+ * Get the unique fields for an entity
+ * @param entity
+ * @return
+ */
+ public static List<Field> getUniqueFields( final Entity entity ){
+
+ final List<Field> uniqueFields = new ArrayList<>(entity.getFields().size());
+
+ for(final Field field: entity.getFields()){
+ if(field.isUnique()){
+ uniqueFields.add( field);
+ }
+ }
+
+ return uniqueFields;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index d55073a..b31cf39 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -322,8 +322,18 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
for (BulkItemResponse response : responses) {
if (response.isFailed()) {
- throw new RuntimeException("Unable to index documents. Errors are :"
- + response.getFailure().getMessage());
+
+ final BulkItemResponse.Failure failure = response.getFailure();
+
+ final String message;
+
+ if(failure != null) {
+ message = "Unable to index documents. Errors are :" + response.getFailure().getMessage();
+ }else{
+ message = "Unable to index documents. Response is :" + response.getResponse();
+ }
+
+ throw new RuntimeException(message);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
index 45218c2..ba0dc1f 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/NoAWSCredsRule.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.persistence.queue;
+import org.junit.Assume;
+import org.junit.internal.runners.model.MultipleFailureException;
import org.junit.rules.TestRule;
import org.junit.runner.Description;
import org.junit.runners.model.Statement;
@@ -49,6 +51,9 @@ public class NoAWSCredsRule implements TestRule {
throw t;
}
+ //do this so our test gets marked as ignored. Not pretty, but it works
+ Assume.assumeTrue( false );
+
}
}
@@ -69,6 +74,18 @@ public class NoAWSCredsRule implements TestRule {
}
}
+ /**
+ * Handle the multiple failure junit trace
+ */
+ if( t instanceof MultipleFailureException ){
+ for(final Throwable failure : ((MultipleFailureException)t).getFailures()){
+ final boolean isMissingCreds = isMissingCredsException( failure );
+
+ if(isMissingCreds){
+ return true;
+ }
+ }
+ }
final Throwable cause = t.getCause();
if ( cause == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index f5e8dd5..68f1a4a 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -133,6 +133,7 @@
<scope>runtime</scope>
</dependency>
+
<dependency>
<groupId>org.apache.usergrid</groupId>
<artifactId>usergrid-core</artifactId>
@@ -290,7 +291,15 @@
<classifier>tests</classifier>
</dependency>
- <dependency>
+ <dependency>
+ <groupId>${project.parent.groupId}</groupId>
+ <artifactId>queue</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.usergrid</groupId>
<artifactId>usergrid-config</artifactId>
<version>${project.version}</version>
@@ -311,6 +320,8 @@
<scope>test</scope>
</dependency>
+
+
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-test</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/03341191/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
index 3c1c645..e6970b5 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/NotifiersServiceIT.java
@@ -17,12 +17,15 @@
package org.apache.usergrid.services.notifications;
import org.apache.commons.io.IOUtils;
+
+import org.apache.usergrid.persistence.queue.NoAWSCredsRule;
import org.apache.usergrid.services.notifications.apns.MockSuccessfulProviderAdapter;
import org.apache.usergrid.persistence.entities.Notifier;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
+import org.junit.Rule;
import org.junit.Test;
import org.apache.usergrid.services.notifications.ConnectionException;
import org.apache.usergrid.services.notifications.NotificationsService;
@@ -48,6 +51,9 @@ public class NotifiersServiceIT extends AbstractServiceIT {
private QueueListener listener;
+ @Rule
+ public NoAWSCredsRule noCredsRule = new NoAWSCredsRule();
+
@Before
public void before() throws Exception {