You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/11/17 23:05:17 UTC
[22/39] usergrid git commit: Added tests and read repair to the guts
of 1.0
Added tests and read repair to the guts of 1.0
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a6705790
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a6705790
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a6705790
Branch: refs/heads/1.x
Commit: a67057909e480c97273c9f6c366643069f24e114
Parents: 7db5769
Author: George Reyes <gr...@apache.org>
Authored: Tue Nov 10 09:06:15 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Tue Nov 10 09:06:15 2015 -0800
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 32 ++++
.../usergrid/persistence/EntityManagerIT.java | 2 +
.../usergrid/services/ServiceRequestIT.java | 169 ++++++++++++++++++-
3 files changed, 202 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6705790/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 4a6e2ea..ecd0f0a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -109,6 +109,7 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static java.util.Arrays.asList;
import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery;
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static org.apache.commons.lang.StringUtils.capitalize;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.usergrid.locking.LockHelper.getUniqueUpdateLock;
@@ -156,6 +157,7 @@ import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtil
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.toStorableBinaryValue;
import static org.apache.usergrid.persistence.cassandra.CassandraService.ALL_COUNT;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.persistence.cassandra.Serializers.le;
import static org.apache.usergrid.persistence.cassandra.Serializers.se;
@@ -538,6 +540,8 @@ public class EntityManagerImpl implements EntityManager {
Object key = createUniqueIndexKey( ownerEntityId, collectionNameInternal, propertyName, propertyValue );
+ //need to fix by asking todd as to why 2.
+ //why is this set to 2?
List<HColumn<ByteBuffer, ByteBuffer>> cols =
cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
false );
@@ -549,12 +553,40 @@ public class EntityManagerImpl implements EntityManager {
}
//shouldn't happen, but it's an error case
+
if ( cols.size() > 1 ) {
logger.error( "INDEX CORRUPTION: More than 1 unique value exists for entities in ownerId {} of type {} on "
+ "property {} with value {}",
new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } );
+
+ //retreive up to 100 columns
+ List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 100,
+ false );
+
+
+
+ //go through it column
+ for ( HColumn<ByteBuffer, ByteBuffer> col : indexingColumns ) {
+ UUID indexCorruptionUuid = ue.fromByteBuffer( col.getName());
+ if (get( indexCorruptionUuid ) == null ) {
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+ Keyspace ko = cass.getApplicationKeyspace( ownerEntityId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, indexCorruptionUuid, timestamp );
+ mutator.execute();
+ cols.remove( col );
+ }
+ else{
+
+ }
+ }
}
+ cols = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
+ false );
+
/**
* Doing this in a loop sucks, but we need to account for possibly having more than 1 entry in the index due
* to corruption. We need to allow them to update, otherwise
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6705790/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
index 5cfc013..b911b3b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
@@ -561,4 +561,6 @@ public class EntityManagerIT extends AbstractCoreIT {
//Not an owner
assertFalse( em.isCollectionMember( createdUser2, "devices", createdDevice ) );
}
+
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6705790/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
index 5b5428a..d89536f 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
@@ -17,7 +17,9 @@
package org.apache.usergrid.services;
+import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -27,15 +29,39 @@ import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.RandomStringUtils;
+
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.ServiceITSuite;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.cassandra.Concurrent;
-
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationOwnerInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.mutation.Mutator;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.services.ServiceParameter.filter;
import static org.apache.usergrid.services.ServiceParameter.parameters;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
@Concurrent()
@@ -76,4 +102,145 @@ public class ServiceRequestIT {
p = filter( path.getParameters(), replaceParameters );
logger.info( "" + p );
}
+
+ //Verify that entity read repair is functioning as intended.
+ @Test
+ public void testRepairOfSingleEntity() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+
+ entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ assertNotNull( entityToBeCorrupted );
+ assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername() );
+
+ }
+
+
+ @Test
+ public void testRepairOfOnlyOneOfTwoColumns() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ //create a new column
+ Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ throw e;
+ }
+
+ //should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
+ //TODO: fix the below so that it fails everytime.
+ //50/50 chance to succeed or fail
+ // assertNull( entityManager
+ // .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+
+
+ //verifies it works now.
+ assertNotNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+ }
}