You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/11/13 18:36:02 UTC
[2/3] usergrid git commit: Added instructions on how to
runUserUniqueIndexCleanup. Replaced UniqueIndexCleanup with
UserUniqueIndexCleanup as it only cleans up users. Changed all
system.out.prints to logger type messages
Added instructions on how to runUserUniqueIndexCleanup.
Replaced UniqueIndexCleanup with UserUniqueIndexCleanup as it only cleans up users.
Changed all system.out.prints to logger type messages
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/15e25dea
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/15e25dea
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/15e25dea
Branch: refs/heads/readRepairForIndexValues
Commit: 15e25dea784257988bbc46a3b3e0185ec09aca23
Parents: a874aa9
Author: George Reyes <gr...@apache.org>
Authored: Fri Nov 13 08:59:14 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Nov 13 08:59:14 2015 -0800
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 18 +-
.../usergrid/tools/UniqueIndexCleanup.java | 473 ------------------
.../usergrid/tools/UserUniqueIndexCleanup.java | 478 ++++++++++++++++++
.../usergrid/tools/UniqueIndexCleanupTest.java | 495 -------------------
.../tools/UserUniqueIndexCleanupTest.java | 490 ++++++++++++++++++
5 files changed, 981 insertions(+), 973 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index ea8c28e..80d9f44 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -571,11 +571,12 @@ public class EntityManagerImpl implements EntityManager {
+ "property {} with value {}",
new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } );
- //retrieve up to 100 columns
- List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 100,
- false );
+ //retrieve ALL DA columns.
+ List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getAllColumns( cass.getApplicationKeyspace( applicationId ),ENTITY_UNIQUE,key,be,be );
+ //Contains entities that weren't deleted but are still in index.
+ //maybe use a set or list so you don't have to keep track of an index.
Entity[] entities = new Entity[cols.size()];
int index = 0;
@@ -586,7 +587,8 @@ public class EntityManagerImpl implements EntityManager {
if (entities[index] == null ) {
deleteUniqueColumn( ownerEntityId, key, indexCorruptionUuid );
- cols.remove( col );
+ //iffy since cols won't have the same values of indexingColumns.
+ //cols.remove( col );
}
else{
index++;
@@ -607,7 +609,7 @@ public class EntityManagerImpl implements EntityManager {
mostRecentEntity = entity;
}
else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
- logger.info("Entities with unique value: "+propertyValue+" has two or more entities with the same modified time."
+ logger.error("Entities with unique value: "+propertyValue+" has two or more entities with the same modified time."
+ "Please manually resolve by query or changing names. ");
}
}
@@ -617,6 +619,12 @@ public class EntityManagerImpl implements EntityManager {
cols = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
false );
+ if ( cols.size() > 1 ) {
+ logger.error( "READ REPAIR FAILURE: More than 1 unique value still exists for entities in ownerId {} of type {} on "
+ + "property {} with value {}",
+ new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } );
+ }
+
/**
* Doing this in a loop sucks, but we need to account for possibly having more than 1 entry in the index due
* to corruption. We need to allow them to update, otherwise
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
deleted file mode 100644
index 64b634c..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ /dev/null
@@ -1,473 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.thrift.TimedOutException;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.thrift.TBaseHelper;
-
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
-import org.apache.usergrid.utils.StringUtils;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import me.prettyprint.cassandra.service.RangeSlicesIterator;
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.ColumnSlice;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.beans.Row;
-import me.prettyprint.hector.api.factory.HFactory;
-import me.prettyprint.hector.api.mutation.Mutator;
-import me.prettyprint.hector.api.query.RangeSlicesQuery;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
-import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-
-
-/**
- * This is a utility to audit all available entity ids in the secondary index. It then checks to see if any index value
- * is not present in the Entity_Index_Entries. If it is not, the value from the index is removed, and a forced re-index
- * is triggered <p/> USERGRID-323 <p/> <p/> UniqueIndexCleanup -app [appid] -col [collectionname]
- *
- * @author tnine
- */
-public class UniqueIndexCleanup extends ToolBase {
-
- /**
- *
- */
- private static final int PAGE_SIZE = 100;
-
-
- private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
-
- private static final String APPLICATION_ARG = "app";
-
- private static final String COLLECTION_ARG = "col";
-
- private static final String ENTITY_UNIQUE_PROPERTY_NAME = "property";
-
- private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
-
-
- @Override
- @SuppressWarnings( "static-access" )
- public Options createOptions() {
-
-
- Options options = new Options();
-
- Option hostOption =
- OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
- .create( "host" );
-
- options.addOption( hostOption );
-
-
- Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
- .withDescription( "application id" ).create( APPLICATION_ARG );
-
-
- options.addOption( appOption );
-
- Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
- .withDescription( "collection name" ).create( COLLECTION_ARG );
-
- options.addOption( collectionOption );
-
- Option entityUniquePropertyName =
- OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_NAME ).hasArg().isRequired( false )
- .withDescription( "Entity Unique Property Name" ).create( ENTITY_UNIQUE_PROPERTY_NAME );
- options.addOption( entityUniquePropertyName );
-
- Option entityUniquePropertyValue =
- OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
- .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
- options.addOption( entityUniquePropertyValue );
-
-
- return options;
- }
-
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
- */
- @Override
- public void runTool( CommandLine line ) throws Exception {
- startSpring();
-
- logger.info( "Starting entity unique index cleanup" );
-
-
- // go through each collection and audit the values
- Keyspace ko = cass.getUsergridApplicationKeyspace();
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- if ( line.hasOption( APPLICATION_ARG ) || line.hasOption( COLLECTION_ARG ) ||
- line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
- deleteInvalidValuesForUniqueProperty( m, line );
- }
- else {
-
- RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
- HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
- //not sure if I trust the lower two settings as it might iterfere with paging or set
- // arbitrary limits and what I want to retrieve.
- //That needs to be verified.
- .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
-
-
- RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
-
- while ( rangeSlicesIterator.hasNext() ) {
- Row rangeSliceValue = rangeSlicesIterator.next();
-
-
- ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
- //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
- // ones return garbage data.
- //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
- String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
- Charset.defaultCharset() ).trim();
-
-
- //defensive programming, don't have to have to parse the string if it doesn't contain users.
- if ( returnedRowKey.contains( "users" ) ) {
-
- String[] parsedRowKey = returnedRowKey.split( ":" );
-
- //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
- if ( parsedRowKey.length > 4 ) {
- parsedRowKey = garbageRowKeyParser( parsedRowKey );
-
- if ( parsedRowKey == null ) {
- System.out.println( returnedRowKey + " is a invalid row key, and unparseable. Skipped..." );
- continue;
- }
- }
- //if the rowkey contains less than four parts then it is completely invalid
- else if ( parsedRowKey.length < 4 ) {
- System.out.println( returnedRowKey + " is a invalid row key and will be skipped" );
- continue;
- }
-
- UUID applicationId = null;
- try {
- applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
- }
- catch ( Exception e ) {
- continue;
- }
- String collectionName = parsedRowKey[1];
- String uniqueValueKey = parsedRowKey[2];
- String uniqueValue = parsedRowKey[3];
-
-
- if ( collectionName.equals( "users" ) ) {
-
- ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
- if ( columnSlice.getColumns().size() != 0 ) {
- List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
- if ( cols.size() == 0 ) {
- deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
- }
- else {
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,returnedRowKey );
- }
- }
- }
- }
- }
- }
- System.out.println( "Completed repair." );
-
- logger.info( "Completed audit of apps" );
- }
-
-
- //Returns a functioning rowkey if it can otherwise returns null
- public String[] garbageRowKeyParser( String[] parsedRowKey ) {
- String[] modifiedRowKey = parsedRowKey.clone();
- while ( modifiedRowKey != null ) {
- if ( modifiedRowKey.length < 4 ) {
- return null;
- }
-
- String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
- if ( recreatedRowKey == null ) {
- recreatedRowKey = "";
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- }
- else {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- break;
- }
- }
- return modifiedRowKey;
- }
-
-
- private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
- for ( int i = 1; i < modifiedRowKey.length; i++ ) {
-
- recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
- if ( i + 1 != modifiedRowKey.length ) {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- }
- }
- modifiedRowKey = recreatedRowKey.split( ":" );
- return modifiedRowKey;
- }
-
-
- private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue ) throws Exception {
- System.out.println( "Found 0 uuid's associated with: " + uniqueValue );
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationId );
- Mutator<ByteBuffer> mutator = createMutator( ko, be );
-
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( mutator, ENTITY_UNIQUE, key, timestamp );
- mutator.execute();
- return;
- }
-
-
- private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue,
- final List<HColumn<ByteBuffer, ByteBuffer>> cols, String rowKey ) throws Exception {
- Boolean cleanup = false;
- EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
- int numberOfColumnsDeleted = 0;
- //these columns all come from the same row key, which means they each belong to the same row key identifier
- //thus mixing and matching them in the below if cases won't matter.
- Entity[] entities = new Entity[cols.size()];
- int numberOfRetrys = 5;
-
- if(cols.size() < 2){
- entities = new Entity[2];
- }
-
- int index = 0;
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- UUID entityId = ue.fromByteBuffer( col.getName() );
-
- if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
- for(int i = 0; i<numberOfRetrys; i++){
- try {
- entities[index] = managementService.getAdminUserEntityByUuid( entityId );
- break;
- }catch(TimedOutException toe){
- Thread.sleep( 2000 );
- }
- }
-
- if ( entities[index] == null ) {
- cleanup = true;
- }
- else {
- index++;
- }
- }
- else {
-
- for(int i = 0; i<numberOfRetrys; i++){
- try {
- entities[index] = em.get( entityId );
- break;
- }catch(TimedOutException toe){
- Thread.sleep( 2000 );
- }
- }
- if ( entities[index] == null ) {
- cleanup = true;
- }
- else {
- index++;
- }
- }
-
- if ( cleanup == true ) {
- numberOfColumnsDeleted++;
- deleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue, entityId );
- cleanup = false;
- }
- }
-
- //this means that the same unique rowkey has two values associated with it
- if(index>2){
- Entity mostRecentEntity = entities[0];
- for(Entity entity: entities){
- if(mostRecentEntity == null){
- System.out.println( "Most Recent entity is null and is being replaced by regular entity" );
- mostRecentEntity = entity;
- }
- if(entity == null){
- System.out.println("Entity we're cycling through is null and we need more new entities");
- continue;
- }
-
- mostRecentEntity = verifyModifiedTimestamp( mostRecentEntity );
- entity = verifyModifiedTimestamp( entity );
-
-
- if(mostRecentEntity.getModified() > entity.getModified()){
- System.out.println("Deleting "+entity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
- System.out.flush();
- em.deleteEntity( entity.getUuid() );
-
- }
- else if (mostRecentEntity.getModified() < entity.getModified()){
- System.out.println("Deleting "+mostRecentEntity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
- System.out.flush();
- em.deleteEntity( mostRecentEntity.getUuid() );
- mostRecentEntity = entity;
- }
- else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
- System.out.println("Entities with unique value: "+uniqueValue+" has two or more entities with the same modified time."
- + "Please manually resolve by query or changing names. ");
- }
- }
- }
-
-
- //a safer way to do this would be to try to do another get and verify there is nothing left in the column
- //instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
- if ( cols.size() == numberOfColumnsDeleted ) {
- deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
- }
- }
-
-
- private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
- Entity entity = unverifiedEntity;
- if(entity !=null && entity.getModified()==null) {
- if(entity.getCreated()!=null){
- System.out.println("Setting modified timestamp to created for comparison purposes");
- entity.setModified( entity.getCreated() );
- return entity;
- }
- else{
- System.out.println("Please delete or remake: "+entity.getUuid());
- System.out.println("Setting timestamps to 1");
- entity.setCreated( 1L );
- entity.setModified( 1L );
- return entity;
- }
- }
- return entity;
- }
-
-
- //really only deletes ones that aren't existant for a specific value
- private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
- UUID applicationId = UUID.fromString( line.getOptionValue( APPLICATION_ARG ) );
- String collectionName = line.getOptionValue( COLLECTION_ARG );
- String uniqueValueKey = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
- String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
-
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
-
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols =
- cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
- false );
-
-
- if ( cols.size() == 0 ) {
- System.out.println( "Zero columns were found for " + key.toString() + ". Will delete rowkey." );
- }
-
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,key.toString() );
- }
-
-
- private String uuidGarbageParser( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- System.out.println( garbageString + " is unparsable" );
- break;
- }
- }
- return stringToBeTruncated;
- }
-
-
- private String uuidStringVerifier( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- return null;
- }
- }
- return stringToBeTruncated;
- }
-
-
- private void deleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue, final UUID entityId )
- throws Exception {
- logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
- System.out.println( "Deleting column uuid: " + entityId.toString() );
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
- Keyspace ko = cass.getApplicationKeyspace( applicationId );
- Mutator<ByteBuffer> mutator = createMutator( ko, be );
-
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( mutator, ENTITY_UNIQUE, key, entityId, timestamp );
- mutator.execute();
- return;
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
new file mode 100644
index 0000000..d95a6e7
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
@@ -0,0 +1,478 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.thrift.TimedOutException;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.thrift.TBaseHelper;
+
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+
+//TODO: write docs so support can run it.
+//TODO: provide support with log4j file and instructions on how to use it.
+/**
+ *
+ *This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users
+ * then it deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row.
+ * If there exists more than one existing column then the one with the most recent timestamp wins and the other is deleted.
+ *
+ *If you want the run the tool on their cluster the following is what you need to do
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here>
+ * -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class UserUniqueIndexCleanup extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( UserUniqueIndexCleanup.class );
+
+ private static final String APPLICATION_ARG = "app";
+
+ private static final String COLLECTION_ARG = "col";
+
+ private static final String ENTITY_UNIQUE_PROPERTY_NAME = "property";
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+
+ Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "application id" ).create( APPLICATION_ARG );
+
+
+ options.addOption( appOption );
+
+ Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
+ .withDescription( "collection name" ).create( COLLECTION_ARG );
+
+ options.addOption( collectionOption );
+
+ Option entityUniquePropertyName =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_NAME ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Name" ).create( ENTITY_UNIQUE_PROPERTY_NAME );
+ options.addOption( entityUniquePropertyName );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique index cleanup" );
+
+
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ if ( line.hasOption( APPLICATION_ARG ) || line.hasOption( COLLECTION_ARG ) ||
+ line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ deleteInvalidValuesForUniqueProperty( m, line );
+ }
+ else {
+//maybe put a byte buffer infront.
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
+ //That needs to be verified.
+ .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+
+
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
+
+
+ ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
+ // ones return garbage data.
+ //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
+ Charset.defaultCharset() ).trim();
+
+
+ //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ if ( returnedRowKey.contains( "users" ) ) {
+
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+
+ //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
+ if ( parsedRowKey.length > 4 ) {
+ parsedRowKey = garbageRowKeyParser( parsedRowKey );
+
+ if ( parsedRowKey == null ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...",returnedRowKey );
+ continue;
+ }
+ }
+ //if the rowkey contains less than four parts then it is completely invalid
+ else if ( parsedRowKey.length < 4 ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...",returnedRowKey );
+ continue;
+ }
+
+ UUID applicationId = null;
+ try {
+ applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ }
+ catch ( Exception e ) {
+ logger.error( "could not parse {} despite earlier parsing. Skipping...",parsedRowKey[0] );
+ continue;
+ }
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
+
+
+ if ( collectionName.equals( "users" ) ) {
+
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ //if ( columnSlice.getColumns().size() != 0 ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+ if ( cols.size() == 0 ) {
+ deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
+ }
+ else {
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,returnedRowKey );
+ }
+ // }
+ }
+ }
+ }
+ }
+ logger.info( "Completed repair successfully" );
+ }
+
+
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 4 ) {
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
+
+ private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue ) throws Exception {
+ logger.debug( "Found 0 uuid's associated with {} Deleting row.",uniqueValue );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, timestamp );
+ mutator.execute();
+ return;
+ }
+
+
+ private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue,
+ final List<HColumn<ByteBuffer, ByteBuffer>> cols, String rowKey ) throws Exception {
+ Boolean cleanup = false;
+ EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
+ int numberOfColumnsDeleted = 0;
+ //these columns all come from the same row key, which means they each belong to the same row key identifier
+ //thus mixing and matching them in the below if cases won't matter.
+ Entity[] entities = new Entity[cols.size()];
+ int numberOfRetrys = 5;
+ int numberOfTimesRetrying = 0;
+
+ int index = 0;
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID entityId = ue.fromByteBuffer( col.getName() );
+
+ //could be the same and just do a entity get
+ if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
+ for(numberOfTimesRetrying = 0; numberOfTimesRetrying<numberOfRetrys; numberOfTimesRetrying++){
+ try {
+ entities[index] = managementService.getAdminUserEntityByUuid( entityId );
+ break;
+ }catch(TimedOutException toe){
+ Thread.sleep( 2000 );
+ }
+ }
+ if(numberOfTimesRetrying == numberOfRetrys ){
+ logger.error( "Tried {} number of times to get the following uuid: {} but failed.Moving on",numberOfRetrys,entityId );
+ continue;
+ }
+
+ if ( entities[index] == null ) {
+ cleanup = true;
+ }
+ else {
+ index++;
+ }
+ }
+ else {
+
+ for(int i = 0; i<numberOfRetrys; i++){
+ try {
+ entities[index] = em.get( entityId );
+ break;
+ }catch(TimedOutException toe){
+ Thread.sleep( 2000 );
+ }
+ }
+ if(numberOfTimesRetrying == numberOfRetrys ){
+ logger.error( "Tried {} number of times to get the following uuid: {} but failed.Moving on",numberOfRetrys,entityId );
+ continue;
+ }
+
+ if ( entities[index] == null ) {
+ cleanup = true;
+ }
+ else {
+ index++;
+ }
+ }
+
+ if ( cleanup == true ) {
+ numberOfColumnsDeleted++;
+ deleteUniqueValue(applicationId, collectionName, uniqueValueKey, uniqueValue, entityId );
+ cleanup = false;
+ }
+ }
+
+ //this means that the same unique rowkey has two values associated with it
+ if(index>2){
+ Entity mostRecentEntity = entities[0];
+ for(Entity entity: entities){
+
+ mostRecentEntity = verifyModifiedTimestamp( mostRecentEntity );
+ entity = verifyModifiedTimestamp( entity );
+
+
+ if(mostRecentEntity.getModified() > entity.getModified()){
+ logger.error( "Deleting {} because it is the older column in the following rowkey: {} ",entity.getUuid().toString(),rowKey );
+ em.deleteEntity( entity.getUuid() );
+ }
+ else if (mostRecentEntity.getModified() < entity.getModified()){
+ logger.error( "Deleting {} because it is the older column in the following rowkey: {} ",mostRecentEntity.getUuid().toString(),rowKey );
+ em.deleteEntity( mostRecentEntity.getUuid() );
+ mostRecentEntity = entity;
+ }
+ else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
+ logger.error( "Entities with unique value: {} contains two or more entities with the same modified time. Please manually fix these"
+ + "entities by deleting one.",rowKey);
+ }
+ }
+ }
+
+
+ //a safer way to do this would be to try to do another get and verify there is nothing left in the column
+ //instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
+ if ( cols.size() == numberOfColumnsDeleted ) {
+ deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
+ }
+ }
+
+
+ private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
+ Entity entity = unverifiedEntity;
+ if(entity !=null && entity.getModified()==null) {
+ if(entity.getCreated()!=null){
+ logger.debug("{} has no modified. Subsituting created timestamp for their modified timestamp.Manually adding one for comparison purposes",entity.getUuid());
+ entity.setModified( entity.getCreated() );
+ return entity;
+ }
+ else{
+ logger.error( "Found no created or modified timestamp. Please remake the following entity: {}."
+ + " Setting both created and modified to 1",entity.getUuid().toString() );
+ entity.setCreated( 1L );
+ entity.setModified( 1L );
+ return entity;
+ }
+ }
+ return entity;
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
+ UUID applicationId = UUID.fromString( line.getOptionValue( APPLICATION_ARG ) );
+ String collectionName = line.getOptionValue( COLLECTION_ARG );
+ String uniqueValueKey = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
+ false );
+
+
+ if ( cols.size() == 0 ) {
+ logger.error( "This row key: {} has zero columns. Deleting...",key.toString() );
+ }
+
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,key.toString() );
+ }
+
+
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ logger.error( "{} is unparsable",garbageString );
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private void deleteUniqueValue( final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue, final UUID entityId )
+ throws Exception {
+ logger.warn( "Entity with id {} did not exist in app {} Deleting", entityId, applicationId );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, entityId, timestamp );
+ mutator.execute();
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
deleted file mode 100644
index 0d921d7..0000000
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.io.File;
-import java.io.FileFilter;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.junit.*;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.RandomStringUtils;
-
-import org.apache.usergrid.ServiceITSetup;
-import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.ServiceITSuite;
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.management.ManagementService;
-import org.apache.usergrid.management.OrganizationOwnerInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.entities.User;
-import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.mutation.MutationResult;
-import me.prettyprint.hector.api.mutation.Mutator;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
-import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-
-/**
- * Created by ApigeeCorporation on 11/2/15.
- */
-public class UniqueIndexCleanupTest {
- static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
-
- int NUM_COLLECTIONS = 10;
- int NUM_ENTITIES = 50;
- int NUM_CONNECTIONS = 3;
-
- @ClassRule
- public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
-
-
- @org.junit.Test
- public void testBasicOperation() throws Exception {
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:9160"
- }, false );
-
- System.out.println( "completed" );
- }
-
- //this test is perfect for the other tool the userCollectionFix tool as this is what I believe they were seeing.
- @Ignore ("WRong test not made for unique index cleanup.")
- public void testRepairOfSingleEntityMissingColumnWrongTool() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,username,username,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
- CassandraService cass = setup.getCassSvc();
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols =
- cass.getColumns( cass.getApplicationKeyspace( applicationInfo.getId() ), ENTITY_UNIQUE, key, null, null,
- 2, false );
-
- Set<UUID> results = new HashSet<UUID>( cols.size() );
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- results.add( ue.fromByteBuffer( col.getName() ) );
- }
-
- UUID uuid = results.iterator().next();
-
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
-
- //Keyspace ko = cass.getUsergridApplicationKeyspace();
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- key = key( applicationInfo.getId(), collectionName, "username", username );
- //addDeleteToMutator( m, ENTITY_UNIQUE, key, uuid, timestamp );
- addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, uuid );
- m.execute();
-
- assertNull( entityManager.getAlias( applicationInfo.getId(), collectionName, username ) );
-
- assertNotNull(entityManager.get( entityToBeCorrupted.getUuid() ));
-
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
-
-
- //here you need to add a delete to the mutator then recheck it and see if the entity is the same as .
- Thread.sleep( 2000 );
- assertNull( entityManager.get( entityToBeCorrupted.getUuid() ) );
-
- //When you come back you also need to emulate the tests to delete what is out of the uuid without any corresponding data.
- //maybe it'll be easier to just do an insert into the EntityUnique row without doint it into any other row and
- //then verify the data like that. Then you don't have to do deletes out of other things.
-
- }
-
- //For this test you need to insert a dummy key with a dummy column that leads to nowhere
- //then run the unique index cleanup.
- //checks for bug when only column doesn't exist make sure to delete the row as well.
-
- //due to the read repair this is no longer a valid test of hte unique index cleanup
- @Ignore
- public void testRepairOfSingleEntity() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name
- addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
-
- m.execute();
-
- //verify that there is no corresponding entity with the uuid or alias provided
- //verify it returns null.
- //assertNull(entityManager.get( testEntityUUID ));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- //The Get above should have repaired the entity allowing it run
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- //fail();
- }catch(DuplicateUniquePropertyExistsException dup){
- fail();
- }
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
-
- //run the cleanup
-// UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
-// uniqueIndexCleanup.startTool( new String[] {
-// "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
-// }, false );
-
-
- entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- assertNotNull( entityToBeCorrupted );
- assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername());
-
- }
-
- //For this test you need to insert a dummy key with a dummy column that leads to nowhere
- //then run the unique index cleanup.
- //checks for bug when only column doesn't exist make sure to delete the row as well.
- // Due to the read repair this is no longer a valid test of unique index cleanup.
- @Test
- @Ignore
- public void testRepairOfMultipleEntities() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- int numberOfEntitiesToCreate = 1000;
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- String[] usernames = new String[numberOfEntitiesToCreate];
-
- int index = 0;
- while(index < numberOfEntitiesToCreate) {
-
- usernames[index]=username+index;
-
- Map<String, Object> userInfo = new HashMap<String, Object>();
- userInfo.put( "username", usernames[index] );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", usernames[index] );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name
- addInsertToMutator( m, ENTITY_UNIQUE, key, testEntityUUID, 0, createTimestamp() );
-
- m.execute();
- index++;
-
- //the below works but not needed for this test.
- //assertNull( entityManager.getAlias("user",username));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
- }
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
- }
-
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
-
- for(String user:usernames ) {
- Map<String, Object> userInfo = new HashMap<String, Object>();
- userInfo.put( "username", user);
- Entity entityToBeCorrupted = entityManager.create( collectionName, userInfo );
-
- assertNotNull( entityToBeCorrupted );
- assertEquals( user, ( ( User ) entityToBeCorrupted ).getUsername() );
- }
- }
-
- @Test
- public void testRepairOfOnlyOneOfTwoColumns() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- //create a new column
- Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
- addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
-
- m.execute();
-
- //verify that there is no corresponding entity with the uuid or alias provided
- //verify it returns null.
- assertNull(entityManager.get( testEntityUUID ));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
- }
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
-
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
-
- //verifies it works now.
- assertNotNull( entityManager
- .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
-
- }
-
- @Test
- public void testStringParsing(){
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
-
- //String garbageString = ")xƐ:�^Q��I�p\\�/2178c690-3a6f-11e4-aec6-48fa705cb26f:users:username:test";
-
- UUID uuid = UUIDUtils.newTimeUUID();
-
- String garbageString = "S2^? <-^Q��%\"�]^S:"+uuid+":users:username:2";
-
- String[] parsedString = garbageString.split( ":" );
-
- String[] cleanedString = uniqueIndexCleanup.garbageRowKeyParser( parsedString );
-
- assertEquals( uuid.toString(),cleanedString[0] );
- assertEquals( "users",cleanedString[1] );
- assertEquals( "username",cleanedString[2] );
- assertEquals( "2",cleanedString[3] );
- }
-
- //POinting at single values is broken now but not entirely used right now anyways.
- //@Ignore
- @Test
- public void testRepairOfOnlyOneOfTwoColumnsWhilePointingAtSingleValue() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- //create a new column
- Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
- addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
-
- m.execute();
-
- //verify that there is no corresponding entity with the uuid or alias provided
- //verify it returns null.
- assertNull(entityManager.get( testEntityUUID ));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
- }
- catch(Exception e) {
- fail( "shouldn't throw something else i think" );
- }
-
- //NEED TO FAIL MORE GRACEFULLY
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort(),
- "-col",collectionName,
- "-app",applicationInfo.getId().toString(),
- "-property","username",
- "-value",username
- }, false );
-
- //verifies it works now.
- assertNotNull( entityManager
- .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
-
- }
-}
-