You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/11/13 18:36:01 UTC
[1/3] usergrid git commit: Added instructions on how to
runUserUniqueIndexCleanup. Replaced UniqueIndexCleanup with
UserUniqueIndexCleanup as it only cleans up users. Changed all
system.out.prints to logger type messages
Repository: usergrid
Updated Branches:
refs/heads/readRepairForIndexValues a874aa901 -> 3a2a88c40
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
new file mode 100644
index 0000000..e08b579
--- /dev/null
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.junit.*;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.RandomStringUtils;
+
+import org.apache.usergrid.ServiceITSetup;
+import org.apache.usergrid.ServiceITSetupImpl;
+import org.apache.usergrid.ServiceITSuite;
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationOwnerInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.mutation.Mutator;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+
+/**
+ * Created by ApigeeCorporation on 11/2/15.
+ */
+public class UserUniqueIndexCleanupTest {
+ static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
+
+ int NUM_COLLECTIONS = 10;
+ int NUM_ENTITIES = 50;
+ int NUM_CONNECTIONS = 3;
+
+ @ClassRule
+ public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
+
+
+ @org.junit.Test
+ public void testBasicOperation() throws Exception {
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:9160"
+ }, false );
+
+ System.out.println( "completed" );
+ }
+
+ //this test is perfect for the other tool the userCollectionFix tool as this is what I believe they were seeing.
+ @Ignore ("WRong test not made for unique index cleanup.")
+ public void testRepairOfSingleEntityMissingColumnWrongTool() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,username,username,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+ CassandraService cass = setup.getCassSvc();
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationInfo.getId() ), ENTITY_UNIQUE, key, null, null,
+ 2, false );
+
+ Set<UUID> results = new HashSet<UUID>( cols.size() );
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ results.add( ue.fromByteBuffer( col.getName() ) );
+ }
+
+ UUID uuid = results.iterator().next();
+
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+
+ //Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ key = key( applicationInfo.getId(), collectionName, "username", username );
+ //addDeleteToMutator( m, ENTITY_UNIQUE, key, uuid, timestamp );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, uuid );
+ m.execute();
+
+ assertNull( entityManager.getAlias( applicationInfo.getId(), collectionName, username ) );
+
+ assertNotNull(entityManager.get( entityToBeCorrupted.getUuid() ));
+
+ //run the cleanup
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
+
+
+ //here you need to add a delete to the mutator then recheck it and see if the entity is the same as .
+ Thread.sleep( 2000 );
+ assertNull( entityManager.get( entityToBeCorrupted.getUuid() ) );
+
+ //When you come back you also need to emulate the tests to delete what is out of the uuid without any corresponding data.
+ //maybe it'll be easier to just do an insert into the EntityUnique row without doint it into any other row and
+ //then verify the data like that. Then you don't have to do deletes out of other things.
+
+ }
+
+ //For this test you need to insert a dummy key with a dummy column that leads to nowhere
+ //then run the unique index cleanup.
+ //checks for bug when only column doesn't exist make sure to delete the row as well.
+
+ //due to the read repair this is no longer a valid test of hte unique index cleanup
+ @Ignore
+ public void testRepairOfSingleEntity() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ //assertNull(entityManager.get( testEntityUUID ));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ //The Get above should have repaired the entity allowing it run
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ //fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+ fail();
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+
+ //run the cleanup
+// UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+// uniqueIndexCleanup.startTool( new String[] {
+// "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+// }, false );
+
+
+ entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ assertNotNull( entityToBeCorrupted );
+ assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername());
+
+ }
+
+ //For this test you need to insert a dummy key with a dummy column that leads to nowhere
+ //then run the unique index cleanup.
+ //checks for bug when only column doesn't exist make sure to delete the row as well.
+ // Due to the read repair this is no longer a valid test of unique index cleanup.
+ @Test
+ @Ignore
+ public void testRepairOfMultipleEntities() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ int numberOfEntitiesToCreate = 1000;
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ String[] usernames = new String[numberOfEntitiesToCreate];
+
+ int index = 0;
+ while(index < numberOfEntitiesToCreate) {
+
+ usernames[index]=username+index;
+
+ Map<String, Object> userInfo = new HashMap<String, Object>();
+ userInfo.put( "username", usernames[index] );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", usernames[index] );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name
+ addInsertToMutator( m, ENTITY_UNIQUE, key, testEntityUUID, 0, createTimestamp() );
+
+ m.execute();
+ index++;
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+ }
+
+ //run the cleanup
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
+
+ for(String user:usernames ) {
+ Map<String, Object> userInfo = new HashMap<String, Object>();
+ userInfo.put( "username", user);
+ Entity entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+
+ assertNotNull( entityToBeCorrupted );
+ assertEquals( user, ( ( User ) entityToBeCorrupted ).getUsername() );
+ }
+ }
+
+ @Test
+ public void testRepairOfOnlyOneOfTwoColumns() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ //create a new column
+ Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+
+ //run the cleanup
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
+
+ //verifies it works now.
+ assertNotNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+ }
+
+ @Test
+ public void testStringParsing(){
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+
+ //String garbageString = ")xƐ:�^Q?�I�p\\�/2178c690-3a6f-11e4-aec6-48fa705cb26f:users:username:test";
+
+ UUID uuid = UUIDUtils.newTimeUUID();
+
+ String garbageString = "S2^? >-^Q��%\"�]^S:"+uuid+":users:username:2";
+
+ String[] parsedString = garbageString.split( ":" );
+
+ String[] cleanedString = userUniqueIndexCleanup.garbageRowKeyParser( parsedString );
+
+ assertEquals( uuid.toString(),cleanedString[0] );
+ assertEquals( "users",cleanedString[1] );
+ assertEquals( "username",cleanedString[2] );
+ assertEquals( "2",cleanedString[3] );
+ }
+
+ //POinting at single values is broken now but not entirely used right now anyways.
+ //@Ignore
+ @Test
+ public void testRepairOfOnlyOneOfTwoColumnsWhilePointingAtSingleValue() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ //create a new column
+ Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e) {
+ fail( "shouldn't throw something else i think" );
+ }
+
+ //NEED TO FAIL MORE GRACEFULLY
+ //run the cleanup
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort(),
+ "-col",collectionName,
+ "-app",applicationInfo.getId().toString(),
+ "-property","username",
+ "-value",username
+ }, false );
+
+ //verifies it works now.
+ assertNotNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+ }
+}
+
[2/3] usergrid git commit: Added instructions on how to
runUserUniqueIndexCleanup. Replaced UniqueIndexCleanup with
UserUniqueIndexCleanup as it only cleans up users. Changed all
system.out.prints to logger type messages
Posted by gr...@apache.org.
Added instructions on how to runUserUniqueIndexCleanup.
Replaced UniqueIndexCleanup with UserUniqueIndexCleanup as it only cleans up users.
Changed all system.out.prints to logger type messages
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/15e25dea
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/15e25dea
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/15e25dea
Branch: refs/heads/readRepairForIndexValues
Commit: 15e25dea784257988bbc46a3b3e0185ec09aca23
Parents: a874aa9
Author: George Reyes <gr...@apache.org>
Authored: Fri Nov 13 08:59:14 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Nov 13 08:59:14 2015 -0800
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 18 +-
.../usergrid/tools/UniqueIndexCleanup.java | 473 ------------------
.../usergrid/tools/UserUniqueIndexCleanup.java | 478 ++++++++++++++++++
.../usergrid/tools/UniqueIndexCleanupTest.java | 495 -------------------
.../tools/UserUniqueIndexCleanupTest.java | 490 ++++++++++++++++++
5 files changed, 981 insertions(+), 973 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index ea8c28e..80d9f44 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -571,11 +571,12 @@ public class EntityManagerImpl implements EntityManager {
+ "property {} with value {}",
new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } );
- //retrieve up to 100 columns
- List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 100,
- false );
+ //retrieve ALL DA columns.
+ List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getAllColumns( cass.getApplicationKeyspace( applicationId ),ENTITY_UNIQUE,key,be,be );
+ //Contains entities that weren't deleted but are still in index.
+ //maybe use a set or list so you don't have to keep track of an index.
Entity[] entities = new Entity[cols.size()];
int index = 0;
@@ -586,7 +587,8 @@ public class EntityManagerImpl implements EntityManager {
if (entities[index] == null ) {
deleteUniqueColumn( ownerEntityId, key, indexCorruptionUuid );
- cols.remove( col );
+ //iffy since cols won't have the same values of indexingColumns.
+ //cols.remove( col );
}
else{
index++;
@@ -607,7 +609,7 @@ public class EntityManagerImpl implements EntityManager {
mostRecentEntity = entity;
}
else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
- logger.info("Entities with unique value: "+propertyValue+" has two or more entities with the same modified time."
+ logger.error("Entities with unique value: "+propertyValue+" has two or more entities with the same modified time."
+ "Please manually resolve by query or changing names. ");
}
}
@@ -617,6 +619,12 @@ public class EntityManagerImpl implements EntityManager {
cols = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
false );
+ if ( cols.size() > 1 ) {
+ logger.error( "READ REPAIR FAILURE: More than 1 unique value still exists for entities in ownerId {} of type {} on "
+ + "property {} with value {}",
+ new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } );
+ }
+
/**
* Doing this in a loop sucks, but we need to account for possibly having more than 1 entry in the index due
* to corruption. We need to allow them to update, otherwise
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
deleted file mode 100644
index 64b634c..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ /dev/null
@@ -1,473 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.thrift.TimedOutException;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.thrift.TBaseHelper;
-
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
-import org.apache.usergrid.utils.StringUtils;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import me.prettyprint.cassandra.service.RangeSlicesIterator;
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.ColumnSlice;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.beans.Row;
-import me.prettyprint.hector.api.factory.HFactory;
-import me.prettyprint.hector.api.mutation.Mutator;
-import me.prettyprint.hector.api.query.RangeSlicesQuery;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
-import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-
-
-/**
- * This is a utility to audit all available entity ids in the secondary index. It then checks to see if any index value
- * is not present in the Entity_Index_Entries. If it is not, the value from the index is removed, and a forced re-index
- * is triggered <p/> USERGRID-323 <p/> <p/> UniqueIndexCleanup -app [appid] -col [collectionname]
- *
- * @author tnine
- */
-public class UniqueIndexCleanup extends ToolBase {
-
- /**
- *
- */
- private static final int PAGE_SIZE = 100;
-
-
- private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
-
- private static final String APPLICATION_ARG = "app";
-
- private static final String COLLECTION_ARG = "col";
-
- private static final String ENTITY_UNIQUE_PROPERTY_NAME = "property";
-
- private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
-
-
- @Override
- @SuppressWarnings( "static-access" )
- public Options createOptions() {
-
-
- Options options = new Options();
-
- Option hostOption =
- OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
- .create( "host" );
-
- options.addOption( hostOption );
-
-
- Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
- .withDescription( "application id" ).create( APPLICATION_ARG );
-
-
- options.addOption( appOption );
-
- Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
- .withDescription( "collection name" ).create( COLLECTION_ARG );
-
- options.addOption( collectionOption );
-
- Option entityUniquePropertyName =
- OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_NAME ).hasArg().isRequired( false )
- .withDescription( "Entity Unique Property Name" ).create( ENTITY_UNIQUE_PROPERTY_NAME );
- options.addOption( entityUniquePropertyName );
-
- Option entityUniquePropertyValue =
- OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
- .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
- options.addOption( entityUniquePropertyValue );
-
-
- return options;
- }
-
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
- */
- @Override
- public void runTool( CommandLine line ) throws Exception {
- startSpring();
-
- logger.info( "Starting entity unique index cleanup" );
-
-
- // go through each collection and audit the values
- Keyspace ko = cass.getUsergridApplicationKeyspace();
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- if ( line.hasOption( APPLICATION_ARG ) || line.hasOption( COLLECTION_ARG ) ||
- line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
- deleteInvalidValuesForUniqueProperty( m, line );
- }
- else {
-
- RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
- HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
- //not sure if I trust the lower two settings as it might iterfere with paging or set
- // arbitrary limits and what I want to retrieve.
- //That needs to be verified.
- .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
-
-
- RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
-
- while ( rangeSlicesIterator.hasNext() ) {
- Row rangeSliceValue = rangeSlicesIterator.next();
-
-
- ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
- //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
- // ones return garbage data.
- //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
- String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
- Charset.defaultCharset() ).trim();
-
-
- //defensive programming, don't have to have to parse the string if it doesn't contain users.
- if ( returnedRowKey.contains( "users" ) ) {
-
- String[] parsedRowKey = returnedRowKey.split( ":" );
-
- //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
- if ( parsedRowKey.length > 4 ) {
- parsedRowKey = garbageRowKeyParser( parsedRowKey );
-
- if ( parsedRowKey == null ) {
- System.out.println( returnedRowKey + " is a invalid row key, and unparseable. Skipped..." );
- continue;
- }
- }
- //if the rowkey contains less than four parts then it is completely invalid
- else if ( parsedRowKey.length < 4 ) {
- System.out.println( returnedRowKey + " is a invalid row key and will be skipped" );
- continue;
- }
-
- UUID applicationId = null;
- try {
- applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
- }
- catch ( Exception e ) {
- continue;
- }
- String collectionName = parsedRowKey[1];
- String uniqueValueKey = parsedRowKey[2];
- String uniqueValue = parsedRowKey[3];
-
-
- if ( collectionName.equals( "users" ) ) {
-
- ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
- if ( columnSlice.getColumns().size() != 0 ) {
- List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
- if ( cols.size() == 0 ) {
- deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
- }
- else {
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,returnedRowKey );
- }
- }
- }
- }
- }
- }
- System.out.println( "Completed repair." );
-
- logger.info( "Completed audit of apps" );
- }
-
-
- //Returns a functioning rowkey if it can otherwise returns null
- public String[] garbageRowKeyParser( String[] parsedRowKey ) {
- String[] modifiedRowKey = parsedRowKey.clone();
- while ( modifiedRowKey != null ) {
- if ( modifiedRowKey.length < 4 ) {
- return null;
- }
-
- String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
- if ( recreatedRowKey == null ) {
- recreatedRowKey = "";
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- }
- else {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- break;
- }
- }
- return modifiedRowKey;
- }
-
-
- private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
- for ( int i = 1; i < modifiedRowKey.length; i++ ) {
-
- recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
- if ( i + 1 != modifiedRowKey.length ) {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- }
- }
- modifiedRowKey = recreatedRowKey.split( ":" );
- return modifiedRowKey;
- }
-
-
- private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue ) throws Exception {
- System.out.println( "Found 0 uuid's associated with: " + uniqueValue );
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationId );
- Mutator<ByteBuffer> mutator = createMutator( ko, be );
-
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( mutator, ENTITY_UNIQUE, key, timestamp );
- mutator.execute();
- return;
- }
-
-
- private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue,
- final List<HColumn<ByteBuffer, ByteBuffer>> cols, String rowKey ) throws Exception {
- Boolean cleanup = false;
- EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
- int numberOfColumnsDeleted = 0;
- //these columns all come from the same row key, which means they each belong to the same row key identifier
- //thus mixing and matching them in the below if cases won't matter.
- Entity[] entities = new Entity[cols.size()];
- int numberOfRetrys = 5;
-
- if(cols.size() < 2){
- entities = new Entity[2];
- }
-
- int index = 0;
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- UUID entityId = ue.fromByteBuffer( col.getName() );
-
- if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
- for(int i = 0; i<numberOfRetrys; i++){
- try {
- entities[index] = managementService.getAdminUserEntityByUuid( entityId );
- break;
- }catch(TimedOutException toe){
- Thread.sleep( 2000 );
- }
- }
-
- if ( entities[index] == null ) {
- cleanup = true;
- }
- else {
- index++;
- }
- }
- else {
-
- for(int i = 0; i<numberOfRetrys; i++){
- try {
- entities[index] = em.get( entityId );
- break;
- }catch(TimedOutException toe){
- Thread.sleep( 2000 );
- }
- }
- if ( entities[index] == null ) {
- cleanup = true;
- }
- else {
- index++;
- }
- }
-
- if ( cleanup == true ) {
- numberOfColumnsDeleted++;
- deleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue, entityId );
- cleanup = false;
- }
- }
-
- //this means that the same unique rowkey has two values associated with it
- if(index>2){
- Entity mostRecentEntity = entities[0];
- for(Entity entity: entities){
- if(mostRecentEntity == null){
- System.out.println( "Most Recent entity is null and is being replaced by regular entity" );
- mostRecentEntity = entity;
- }
- if(entity == null){
- System.out.println("Entity we're cycling through is null and we need more new entities");
- continue;
- }
-
- mostRecentEntity = verifyModifiedTimestamp( mostRecentEntity );
- entity = verifyModifiedTimestamp( entity );
-
-
- if(mostRecentEntity.getModified() > entity.getModified()){
- System.out.println("Deleting "+entity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
- System.out.flush();
- em.deleteEntity( entity.getUuid() );
-
- }
- else if (mostRecentEntity.getModified() < entity.getModified()){
- System.out.println("Deleting "+mostRecentEntity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
- System.out.flush();
- em.deleteEntity( mostRecentEntity.getUuid() );
- mostRecentEntity = entity;
- }
- else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
- System.out.println("Entities with unique value: "+uniqueValue+" has two or more entities with the same modified time."
- + "Please manually resolve by query or changing names. ");
- }
- }
- }
-
-
- //a safer way to do this would be to try to do another get and verify there is nothing left in the column
- //instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
- if ( cols.size() == numberOfColumnsDeleted ) {
- deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
- }
- }
-
-
- private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
- Entity entity = unverifiedEntity;
- if(entity !=null && entity.getModified()==null) {
- if(entity.getCreated()!=null){
- System.out.println("Setting modified timestamp to created for comparison purposes");
- entity.setModified( entity.getCreated() );
- return entity;
- }
- else{
- System.out.println("Please delete or remake: "+entity.getUuid());
- System.out.println("Setting timestamps to 1");
- entity.setCreated( 1L );
- entity.setModified( 1L );
- return entity;
- }
- }
- return entity;
- }
-
-
- //really only deletes ones that aren't existant for a specific value
- private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
- UUID applicationId = UUID.fromString( line.getOptionValue( APPLICATION_ARG ) );
- String collectionName = line.getOptionValue( COLLECTION_ARG );
- String uniqueValueKey = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
- String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
-
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
-
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols =
- cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
- false );
-
-
- if ( cols.size() == 0 ) {
- System.out.println( "Zero columns were found for " + key.toString() + ". Will delete rowkey." );
- }
-
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,key.toString() );
- }
-
-
- private String uuidGarbageParser( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- System.out.println( garbageString + " is unparsable" );
- break;
- }
- }
- return stringToBeTruncated;
- }
-
-
- private String uuidStringVerifier( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- return null;
- }
- }
- return stringToBeTruncated;
- }
-
-
- private void deleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue, final UUID entityId )
- throws Exception {
- logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
- System.out.println( "Deleting column uuid: " + entityId.toString() );
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
- Keyspace ko = cass.getApplicationKeyspace( applicationId );
- Mutator<ByteBuffer> mutator = createMutator( ko, be );
-
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( mutator, ENTITY_UNIQUE, key, entityId, timestamp );
- mutator.execute();
- return;
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
new file mode 100644
index 0000000..d95a6e7
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
@@ -0,0 +1,478 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.thrift.TimedOutException;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.thrift.TBaseHelper;
+
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+
+//TODO: write docs so support can run it.
+//TODO: provide support with log4j file and instructions on how to use it.
+/**
+ *
+ *This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users
+ * then it deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row.
+ * If there exists more than one existing column then the one with the most recent timestamp wins and the other is deleted.
+ *
+ *If you want the run the tool on their cluster the following is what you need to do
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here>
+ * -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class UserUniqueIndexCleanup extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( UserUniqueIndexCleanup.class );
+
+ private static final String APPLICATION_ARG = "app";
+
+ private static final String COLLECTION_ARG = "col";
+
+ private static final String ENTITY_UNIQUE_PROPERTY_NAME = "property";
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+
+ Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "application id" ).create( APPLICATION_ARG );
+
+
+ options.addOption( appOption );
+
+ Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
+ .withDescription( "collection name" ).create( COLLECTION_ARG );
+
+ options.addOption( collectionOption );
+
+ Option entityUniquePropertyName =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_NAME ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Name" ).create( ENTITY_UNIQUE_PROPERTY_NAME );
+ options.addOption( entityUniquePropertyName );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique index cleanup" );
+
+
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ if ( line.hasOption( APPLICATION_ARG ) || line.hasOption( COLLECTION_ARG ) ||
+ line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ deleteInvalidValuesForUniqueProperty( m, line );
+ }
+ else {
+//maybe put a byte buffer infront.
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
+ //That needs to be verified.
+ .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+
+
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
+
+
+ ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
+ // ones return garbage data.
+ //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
+ Charset.defaultCharset() ).trim();
+
+
+ //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ if ( returnedRowKey.contains( "users" ) ) {
+
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+
+ //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
+ if ( parsedRowKey.length > 4 ) {
+ parsedRowKey = garbageRowKeyParser( parsedRowKey );
+
+ if ( parsedRowKey == null ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...",returnedRowKey );
+ continue;
+ }
+ }
+ //if the rowkey contains less than four parts then it is completely invalid
+ else if ( parsedRowKey.length < 4 ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...",returnedRowKey );
+ continue;
+ }
+
+ UUID applicationId = null;
+ try {
+ applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ }
+ catch ( Exception e ) {
+ logger.error( "could not parse {} despite earlier parsing. Skipping...",parsedRowKey[0] );
+ continue;
+ }
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
+
+
+ if ( collectionName.equals( "users" ) ) {
+
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ //if ( columnSlice.getColumns().size() != 0 ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+ if ( cols.size() == 0 ) {
+ deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
+ }
+ else {
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,returnedRowKey );
+ }
+ // }
+ }
+ }
+ }
+ }
+ logger.info( "Completed repair successfully" );
+ }
+
+
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 4 ) {
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
+
+ private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue ) throws Exception {
+ logger.debug( "Found 0 uuid's associated with {} Deleting row.",uniqueValue );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, timestamp );
+ mutator.execute();
+ return;
+ }
+
+
+ private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue,
+ final List<HColumn<ByteBuffer, ByteBuffer>> cols, String rowKey ) throws Exception {
+ Boolean cleanup = false;
+ EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
+ int numberOfColumnsDeleted = 0;
+ //these columns all come from the same row key, which means they each belong to the same row key identifier
+ //thus mixing and matching them in the below if cases won't matter.
+ Entity[] entities = new Entity[cols.size()];
+ int numberOfRetrys = 5;
+ int numberOfTimesRetrying = 0;
+
+ int index = 0;
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID entityId = ue.fromByteBuffer( col.getName() );
+
+ //could be the same and just do a entity get
+ if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
+ for(numberOfTimesRetrying = 0; numberOfTimesRetrying<numberOfRetrys; numberOfTimesRetrying++){
+ try {
+ entities[index] = managementService.getAdminUserEntityByUuid( entityId );
+ break;
+ }catch(TimedOutException toe){
+ Thread.sleep( 2000 );
+ }
+ }
+ if(numberOfTimesRetrying == numberOfRetrys ){
+ logger.error( "Tried {} number of times to get the following uuid: {} but failed.Moving on",numberOfRetrys,entityId );
+ continue;
+ }
+
+ if ( entities[index] == null ) {
+ cleanup = true;
+ }
+ else {
+ index++;
+ }
+ }
+ else {
+
+ for(int i = 0; i<numberOfRetrys; i++){
+ try {
+ entities[index] = em.get( entityId );
+ break;
+ }catch(TimedOutException toe){
+ Thread.sleep( 2000 );
+ }
+ }
+ if(numberOfTimesRetrying == numberOfRetrys ){
+ logger.error( "Tried {} number of times to get the following uuid: {} but failed.Moving on",numberOfRetrys,entityId );
+ continue;
+ }
+
+ if ( entities[index] == null ) {
+ cleanup = true;
+ }
+ else {
+ index++;
+ }
+ }
+
+ if ( cleanup == true ) {
+ numberOfColumnsDeleted++;
+ deleteUniqueValue(applicationId, collectionName, uniqueValueKey, uniqueValue, entityId );
+ cleanup = false;
+ }
+ }
+
+ //this means that the same unique rowkey has two values associated with it
+ if(index>2){
+ Entity mostRecentEntity = entities[0];
+ for(Entity entity: entities){
+
+ mostRecentEntity = verifyModifiedTimestamp( mostRecentEntity );
+ entity = verifyModifiedTimestamp( entity );
+
+
+ if(mostRecentEntity.getModified() > entity.getModified()){
+ logger.error( "Deleting {} because it is the older column in the following rowkey: {} ",entity.getUuid().toString(),rowKey );
+ em.deleteEntity( entity.getUuid() );
+ }
+ else if (mostRecentEntity.getModified() < entity.getModified()){
+ logger.error( "Deleting {} because it is the older column in the following rowkey: {} ",mostRecentEntity.getUuid().toString(),rowKey );
+ em.deleteEntity( mostRecentEntity.getUuid() );
+ mostRecentEntity = entity;
+ }
+ else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
+ logger.error( "Entities with unique value: {} contains two or more entities with the same modified time. Please manually fix these"
+ + "entities by deleting one.",rowKey);
+ }
+ }
+ }
+
+
+ //a safer way to do this would be to try to do another get and verify there is nothing left in the column
+ //instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
+ if ( cols.size() == numberOfColumnsDeleted ) {
+ deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
+ }
+ }
+
+
+ private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
+ Entity entity = unverifiedEntity;
+ if(entity !=null && entity.getModified()==null) {
+ if(entity.getCreated()!=null){
+ logger.debug("{} has no modified. Subsituting created timestamp for their modified timestamp.Manually adding one for comparison purposes",entity.getUuid());
+ entity.setModified( entity.getCreated() );
+ return entity;
+ }
+ else{
+ logger.error( "Found no created or modified timestamp. Please remake the following entity: {}."
+ + " Setting both created and modified to 1",entity.getUuid().toString() );
+ entity.setCreated( 1L );
+ entity.setModified( 1L );
+ return entity;
+ }
+ }
+ return entity;
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
+ UUID applicationId = UUID.fromString( line.getOptionValue( APPLICATION_ARG ) );
+ String collectionName = line.getOptionValue( COLLECTION_ARG );
+ String uniqueValueKey = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
+ false );
+
+
+ if ( cols.size() == 0 ) {
+ logger.error( "This row key: {} has zero columns. Deleting...",key.toString() );
+ }
+
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,key.toString() );
+ }
+
+
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ logger.error( "{} is unparsable",garbageString );
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private void deleteUniqueValue( final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue, final UUID entityId )
+ throws Exception {
+ logger.warn( "Entity with id {} did not exist in app {} Deleting", entityId, applicationId );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, entityId, timestamp );
+ mutator.execute();
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
deleted file mode 100644
index 0d921d7..0000000
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.io.File;
-import java.io.FileFilter;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.junit.*;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.RandomStringUtils;
-
-import org.apache.usergrid.ServiceITSetup;
-import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.ServiceITSuite;
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.management.ManagementService;
-import org.apache.usergrid.management.OrganizationOwnerInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.entities.User;
-import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.mutation.MutationResult;
-import me.prettyprint.hector.api.mutation.Mutator;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
-import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-
-/**
- * Created by ApigeeCorporation on 11/2/15.
- */
-public class UniqueIndexCleanupTest {
- static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
-
- int NUM_COLLECTIONS = 10;
- int NUM_ENTITIES = 50;
- int NUM_CONNECTIONS = 3;
-
- @ClassRule
- public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
-
-
- @org.junit.Test
- public void testBasicOperation() throws Exception {
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:9160"
- }, false );
-
- System.out.println( "completed" );
- }
-
- //this test is perfect for the other tool the userCollectionFix tool as this is what I believe they were seeing.
- @Ignore ("WRong test not made for unique index cleanup.")
- public void testRepairOfSingleEntityMissingColumnWrongTool() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,username,username,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
- CassandraService cass = setup.getCassSvc();
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols =
- cass.getColumns( cass.getApplicationKeyspace( applicationInfo.getId() ), ENTITY_UNIQUE, key, null, null,
- 2, false );
-
- Set<UUID> results = new HashSet<UUID>( cols.size() );
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- results.add( ue.fromByteBuffer( col.getName() ) );
- }
-
- UUID uuid = results.iterator().next();
-
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
-
- //Keyspace ko = cass.getUsergridApplicationKeyspace();
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- key = key( applicationInfo.getId(), collectionName, "username", username );
- //addDeleteToMutator( m, ENTITY_UNIQUE, key, uuid, timestamp );
- addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, uuid );
- m.execute();
-
- assertNull( entityManager.getAlias( applicationInfo.getId(), collectionName, username ) );
-
- assertNotNull(entityManager.get( entityToBeCorrupted.getUuid() ));
-
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
-
-
- //here you need to add a delete to the mutator then recheck it and see if the entity is the same as .
- Thread.sleep( 2000 );
- assertNull( entityManager.get( entityToBeCorrupted.getUuid() ) );
-
- //When you come back you also need to emulate the tests to delete what is out of the uuid without any corresponding data.
- //maybe it'll be easier to just do an insert into the EntityUnique row without doint it into any other row and
- //then verify the data like that. Then you don't have to do deletes out of other things.
-
- }
-
- //For this test you need to insert a dummy key with a dummy column that leads to nowhere
- //then run the unique index cleanup.
- //checks for bug when only column doesn't exist make sure to delete the row as well.
-
- //due to the read repair this is no longer a valid test of hte unique index cleanup
- @Ignore
- public void testRepairOfSingleEntity() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name
- addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
-
- m.execute();
-
- //verify that there is no corresponding entity with the uuid or alias provided
- //verify it returns null.
- //assertNull(entityManager.get( testEntityUUID ));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- //The Get above should have repaired the entity allowing it run
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- //fail();
- }catch(DuplicateUniquePropertyExistsException dup){
- fail();
- }
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
-
- //run the cleanup
-// UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
-// uniqueIndexCleanup.startTool( new String[] {
-// "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
-// }, false );
-
-
- entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- assertNotNull( entityToBeCorrupted );
- assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername());
-
- }
-
- //For this test you need to insert a dummy key with a dummy column that leads to nowhere
- //then run the unique index cleanup.
- //checks for bug when only column doesn't exist make sure to delete the row as well.
- // Due to the read repair this is no longer a valid test of unique index cleanup.
- @Test
- @Ignore
- public void testRepairOfMultipleEntities() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- int numberOfEntitiesToCreate = 1000;
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- String[] usernames = new String[numberOfEntitiesToCreate];
-
- int index = 0;
- while(index < numberOfEntitiesToCreate) {
-
- usernames[index]=username+index;
-
- Map<String, Object> userInfo = new HashMap<String, Object>();
- userInfo.put( "username", usernames[index] );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", usernames[index] );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name
- addInsertToMutator( m, ENTITY_UNIQUE, key, testEntityUUID, 0, createTimestamp() );
-
- m.execute();
- index++;
-
- //the below works but not needed for this test.
- //assertNull( entityManager.getAlias("user",username));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
- }
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
- }
-
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
-
- for(String user:usernames ) {
- Map<String, Object> userInfo = new HashMap<String, Object>();
- userInfo.put( "username", user);
- Entity entityToBeCorrupted = entityManager.create( collectionName, userInfo );
-
- assertNotNull( entityToBeCorrupted );
- assertEquals( user, ( ( User ) entityToBeCorrupted ).getUsername() );
- }
- }
-
- @Test
- public void testRepairOfOnlyOneOfTwoColumns() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- //create a new column
- Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
- addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
-
- m.execute();
-
- //verify that there is no corresponding entity with the uuid or alias provided
- //verify it returns null.
- assertNull(entityManager.get( testEntityUUID ));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
- }
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
-
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
-
- //verifies it works now.
- assertNotNull( entityManager
- .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
-
- }
-
- @Test
- public void testStringParsing(){
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
-
- //String garbageString = ")xƐ:�^Q��I�p\\�/2178c690-3a6f-11e4-aec6-48fa705cb26f:users:username:test";
-
- UUID uuid = UUIDUtils.newTimeUUID();
-
- String garbageString = "S2^? <-^Q��%\"�]^S:"+uuid+":users:username:2";
-
- String[] parsedString = garbageString.split( ":" );
-
- String[] cleanedString = uniqueIndexCleanup.garbageRowKeyParser( parsedString );
-
- assertEquals( uuid.toString(),cleanedString[0] );
- assertEquals( "users",cleanedString[1] );
- assertEquals( "username",cleanedString[2] );
- assertEquals( "2",cleanedString[3] );
- }
-
- //POinting at single values is broken now but not entirely used right now anyways.
- //@Ignore
- @Test
- public void testRepairOfOnlyOneOfTwoColumnsWhilePointingAtSingleValue() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- //create a new column
- Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
- addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
-
- m.execute();
-
- //verify that there is no corresponding entity with the uuid or alias provided
- //verify it returns null.
- assertNull(entityManager.get( testEntityUUID ));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
- }
- catch(Exception e) {
- fail( "shouldn't throw something else i think" );
- }
-
- //NEED TO FAIL MORE GRACEFULLY
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort(),
- "-col",collectionName,
- "-app",applicationInfo.getId().toString(),
- "-property","username",
- "-value",username
- }, false );
-
- //verifies it works now.
- assertNotNull( entityManager
- .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
-
- }
-}
-
[3/3] usergrid git commit: Added some additional comments and final
look over of entity manager read repair.
Posted by gr...@apache.org.
Added some additional comments and final look over of entity manager read repair.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3a2a88c4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3a2a88c4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3a2a88c4
Branch: refs/heads/readRepairForIndexValues
Commit: 3a2a88c405ccf6a7a7fa1fd9d19e8a1067a2f3f0
Parents: 15e25de
Author: George Reyes <gr...@apache.org>
Authored: Fri Nov 13 09:35:57 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Nov 13 09:35:57 2015 -0800
----------------------------------------------------------------------
.../usergrid/persistence/cassandra/EntityManagerImpl.java | 2 --
.../java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java | 6 ++----
2 files changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3a2a88c4/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 80d9f44..38a80a0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -587,8 +587,6 @@ public class EntityManagerImpl implements EntityManager {
if (entities[index] == null ) {
deleteUniqueColumn( ownerEntityId, key, indexCorruptionUuid );
- //iffy since cols won't have the same values of indexingColumns.
- //cols.remove( col );
}
else{
index++;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3a2a88c4/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
index d95a6e7..aec4266 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
@@ -55,8 +55,6 @@ import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-//TODO: write docs so support can run it.
-//TODO: provide support with log4j file and instructions on how to use it.
/**
*
*This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users
@@ -64,11 +62,11 @@ import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
* If there exists more than one existing column then the one with the most recent timestamp wins and the other is deleted.
*
*If you want the run the tool on their cluster the following is what you need to do
- * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here> > log.txt
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host <cassandra_host_here> > log.txt
*
* if there is a specific value you want to run the tool on then you need the following
- * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here>
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host <cassandra_host_here>
* -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value <unique_property_value> > log.txt
*
* @author grey