You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/12/12 20:39:23 UTC
[1/4] usergrid git commit: Changed existing tools to look through and
see if there is a value in the ENTITY_* type index and see if it also exists
in entity_properties. Requires a .json file with users email array as input.
Repository: usergrid
Updated Branches:
refs/heads/unique_index_logging ddbba7bc5 -> 9e526ba5b
Changed existing tools to look through and see if there is a value in the ENTITY_* type index and see if it also exists in entity_properties. Requires a .json file with users email array as input.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5a964160
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5a964160
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5a964160
Branch: refs/heads/unique_index_logging
Commit: 5a96416071522973e6bac07a62b4372e2f502f24
Parents: ddbba7b
Author: George Reyes <gr...@apache.org>
Authored: Fri Dec 11 09:14:48 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Dec 11 09:14:48 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/ManagementUserAudit.java | 311 +++----------------
.../tools/ManagementUserIndexAudit.java | 289 ++---------------
2 files changed, 77 insertions(+), 523 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5a964160/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
index 4de6391..d1eea62 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
@@ -17,13 +17,13 @@
package org.apache.usergrid.tools;
+import java.io.File;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
import java.util.List;
-import java.util.Map;
-import java.util.Set;
import java.util.UUID;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,37 +31,18 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.thrift.TBaseHelper;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.entities.User;
-import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import com.google.common.collect.BiMap;
-
-import me.prettyprint.cassandra.service.RangeSlicesIterator;
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.ColumnSlice;
+import org.apache.usergrid.persistence.Query;
+
import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.beans.Row;
-import me.prettyprint.hector.api.factory.HFactory;
-import me.prettyprint.hector.api.mutation.Mutator;
-import me.prettyprint.hector.api.query.RangeSlicesQuery;
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.Results.Level.REFS;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
/**
@@ -92,7 +73,7 @@ public class ManagementUserAudit extends ToolBase {
private static final Logger logger = LoggerFactory.getLogger( ManagementUserAudit.class );
- private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "file";
@Override
@@ -109,8 +90,8 @@ public class ManagementUserAudit extends ToolBase {
options.addOption( hostOption );
Option entityUniquePropertyValue =
- OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
- .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( true )
+ .withDescription( "file path" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
options.addOption( entityUniquePropertyValue );
@@ -130,265 +111,61 @@ public class ManagementUserAudit extends ToolBase {
logger.info( "Starting entity unique checker" );
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
- // go through each collection and audit the values
- Keyspace ko = cass.getUsergridApplicationKeyspace();
- Mutator<ByteBuffer> m = createMutator( ko, be );
+ ObjectMapper objectMapper = new ObjectMapper( );
- if ( line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
- deleteInvalidValuesForUniqueProperty( m, line );
- }
- else {
- //maybe put a byte buffer infront.
- RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
- HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
- //not sure if I trust the lower two settings as it might iterfere with paging or set
- // arbitrary limits and what I want to retrieve.
- //That needs to be verified.
- .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
-
-
- RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
-
- while ( rangeSlicesIterator.hasNext() ) {
- Row rangeSliceValue = rangeSlicesIterator.next();
-
-
- ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
- //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
- // ones return garbage data.
- //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
- String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
- Charset.defaultCharset() ).trim();
-
-
- //defensive programming, don't have to have to parse the string if it doesn't contain users.
- if (returnedRowKey.contains("email") && returnedRowKey.contains( "users" ) && returnedRowKey.contains( MANAGEMENT_APPLICATION_ID.toString() )) {
-
- String[] parsedRowKey = returnedRowKey.split( ":" );
-
- //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
- if ( parsedRowKey.length > 4 ) {
- parsedRowKey = garbageRowKeyParser( parsedRowKey );
-
- if ( parsedRowKey == null ) {
- logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
- continue;
- }
- }
- //if the rowkey contains less than four parts then it is completely invalid
- else if ( parsedRowKey.length < 4 ) {
- logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
- continue;
- }
-
- UUID applicationId = null;
- try {
- applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
- }
- catch ( Exception e ) {
- logger.error( "could not parse {} despite earlier parsing. Skipping...", parsedRowKey[0] );
- continue;
- }
- String collectionName = parsedRowKey[1];
- String uniqueValueKey = parsedRowKey[2];
- String uniqueValue = parsedRowKey[3];
-
-
- if ( collectionName.equals( "users" ) ) {
-
- ColumnSlice<ByteBuffer, ByteBuffer> columnSlice=rangeSliceValue.getColumnSlice();
- //if ( columnSlice.getColumns().size() != 0 ) {
- List<HColumn<ByteBuffer, ByteBuffer>> cols=columnSlice.getColumns();
-
- entityStateLogger( uniqueValue, cols );
- }
- }
- }
- }
- logger.info( "Completed logging successfully" );
- }
+ File jsonObjectFile = new File(line.getOptionValue( "file" ));
+ JsonNode node = objectMapper.readTree( jsonObjectFile );
- //Returns a functioning rowkey if it can otherwise returns null
- public String[] garbageRowKeyParser( String[] parsedRowKey ) {
- String[] modifiedRowKey = parsedRowKey.clone();
- while ( modifiedRowKey != null ) {
- if ( modifiedRowKey.length < 4 ) {
- return null;
- }
+ JsonNode users = node.get( "user" );
- String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
- if ( recreatedRowKey == null ) {
- recreatedRowKey = "";
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- }
- else {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- break;
- }
- }
- return modifiedRowKey;
- }
+ for(JsonNode email:users) {
+ String extractedEmail = email.get( "name" ).getTextValue();
- private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
- for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+ Query query = new Query();
+ query.setEntityType( "user" );
+ query.addEqualityFilter( "email", extractedEmail );
+ //maybe this could be changed to detect duplicates
+ query.setLimit( 1 );
+ query.setResultsLevel( REFS );
- recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
- if ( i + 1 != modifiedRowKey.length ) {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- }
- }
- modifiedRowKey = recreatedRowKey.split( ":" );
- return modifiedRowKey;
- }
- private void entityStateLogger( final String uniqueValue, final List<HColumn<ByteBuffer, ByteBuffer>> cols ) throws Exception {
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "users";
+ String uniqueValueKey = "email";
+ String uniqueValue = extractedEmail;
- UserInfo userInfo = null;
- EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
- try {
- userInfo = managementService.getAdminUserByEmail( uniqueValue );
- }catch(Exception e){
- logger.error("threw exception when looking up email: {}",uniqueValue);
- e.printStackTrace();
- }
- if(userInfo==null) {
- if(cols!=null){
- if(cols.size()>1){
- for(HColumn<ByteBuffer, ByteBuffer> col : cols) {
- logger.warn( "This uuid: {} is associated with this duplicated email {}", ue.fromByteBuffer( col.getName()),uniqueValue );
- }
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = cass.getAllColumns( cass.getApplicationKeyspace( applicationId ),ENTITY_UNIQUE,key,be,be );
+
+ if ( cols.size() == 1 ) {
+ UUID uuid = null;
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ uuid = ue.fromByteBuffer( col.getName());
}
- if(cols.size()==1){
- logger.error( "Management user with uuid: {} and email: {} is broken.",ue.fromByteBuffer( cols.get( 0 ).getName()), uniqueValue );
+ if ( em.get( uuid ) == null ) {
+ logger.error( "Email: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail,uuid);
}
- else{
- logger.error( "Management user with email: {} is broken and has no uuid's associated with it",uniqueValue );
+ else {
+ logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES", extractedEmail,uuid);
}
}
- }
- else {
- Results orgResults =
- em.getCollection( new SimpleEntityRef( User.ENTITY_TYPE, userInfo.getUuid() ), "groups", null, 10000, Results.Level.REFS,
- false );
-
- if(orgResults.getRefs().size() == 1){
- EntityRef orgRef = orgResults.getRef();
- orgVerification( uniqueValue, em, orgRef, orgRef.getUuid(),
- "Management user with email: {} is present but cannot login due to missing their only organization: {}" );
- }
- else {
- for ( EntityRef orgRef : orgResults.getRefs() ) {
- orgVerification( uniqueValue, em, orgRef, orgRef.getUuid(),
- "Management user with email: {} is present with multiple orgs but is missing the following orgUUID: {}" );
-
+ else{
+ if(cols.size() == 0) {
+ logger.error( "Email: {} doesn't exist in ENTITY_UNIQUE.", extractedEmail );
}
- }
- }
-
- }
-
-
- private void orgVerification( final String uniqueValue, final EntityManager em, final EntityRef orgRef,
- final UUID uuid, final String s2 ) throws Exception {
- try {
- em.getDictionaryAsMap( orgRef, "orgProperties" );
- OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( uuid );
-
- if(organizationInfo == null) {
- logger.error( "The following email works: {} but the orgUUID: {} returns null.",uniqueValue,uuid );
- }
- else {
-
- Object[] loggingObject = new Object[3];
- loggingObject[0] = uniqueValue;
- loggingObject[1] = organizationInfo.getName();
- loggingObject[2] = organizationInfo.getUuid();
-
- logger.info( "The following email works: {} with the following orgname: {} and orgUUID: {}",
- loggingObject );
- }
- }
- catch ( EntityNotFoundException enfe ) {
- logger.error( s2, uniqueValue, uuid );
-
- BiMap<UUID,String> applicationBiMap = managementService.getApplicationsForOrganization( uuid );
- if(applicationBiMap==null || applicationBiMap.size() == 0){
- logger.error("Applications didn't return for orgUUID: {}",uuid);
- }
- else if (applicationBiMap.size()>0){
- for ( Map.Entry<UUID, String> app : applicationBiMap.entrySet() ) {
- Object[] loggingObject = new Object[3];
- loggingObject[0] = app.getKey();
- loggingObject[1] = app.getValue();
- loggingObject[2] = uuid;
- logger.info( "Application with uuid: {} and name: {} was returned from orgUUID: {} ",loggingObject );
-
+ else{
+ logger.error("Email: {} has {} number of duplicate columns in ENTITY_UNIQUE",extractedEmail,cols.size());
}
}
-
}
+ logger.info( "Completed logging successfully" );
}
-
-
- //really only deletes ones that aren't existant for a specific value
- private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
- UUID applicationId = MANAGEMENT_APPLICATION_ID;
- String collectionName = "users"; //line.getOptionValue( COLLECTION_ARG );
- String uniqueValueKey = "email"; //line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
- String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
-
- //PLEASE ADD VERIFICATION.
-
- Object key = key( applicationId, collectionName,"email", uniqueValue );
-
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols = cass.getColumns( cass.getApplicationKeyspace( applicationId), ENTITY_UNIQUE, key, null, null, 1000,
- false );
-
-
- if ( cols.size() == 0 ) {
- logger.error( "This row key: {} has zero columns", key.toString() );
- }
-
- entityStateLogger( uniqueValue, cols );
- }
-
-
- private String uuidGarbageParser( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- logger.error( "{} is unparsable", garbageString );
- break;
- }
- }
- return stringToBeTruncated;
- }
-
-
- private String uuidStringVerifier( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- return null;
- }
- }
- return stringToBeTruncated;
- }
-
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5a964160/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexAudit.java
index 06c4895..4ab2ad0 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexAudit.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexAudit.java
@@ -17,14 +17,11 @@
package org.apache.usergrid.tools;
+import java.io.File;
import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,42 +29,19 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.thrift.TBaseHelper;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
-import org.apache.usergrid.persistence.entities.User;
-import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
-import org.apache.usergrid.utils.UUIDUtils;
-import com.fasterxml.uuid.impl.UUIDUtil;
-import com.google.common.collect.BiMap;
-
-import me.prettyprint.cassandra.service.RangeSlicesIterator;
import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.ColumnSlice;
-import me.prettyprint.hector.api.beans.DynamicComposite;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.beans.Row;
-import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
-import me.prettyprint.hector.api.query.RangeSlicesQuery;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.Results.Level.REFS;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.dce;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
/**
@@ -90,15 +64,9 @@ import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
*/
public class ManagementUserIndexAudit extends ToolBase {
- /**
- *
- */
- private static final int PAGE_SIZE = 100;
-
-
private static final Logger logger = LoggerFactory.getLogger( ManagementUserIndexAudit.class );
- private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "file";
@Override
@@ -115,8 +83,8 @@ public class ManagementUserIndexAudit extends ToolBase {
options.addOption( hostOption );
Option entityUniquePropertyValue =
- OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
- .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( true )
+ .withDescription( "file path" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
options.addOption( entityUniquePropertyValue );
@@ -141,233 +109,42 @@ public class ManagementUserIndexAudit extends ToolBase {
Keyspace ko = cass.getUsergridApplicationKeyspace();
Mutator<ByteBuffer> m = createMutator( ko, be );
- if ( line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
- deleteInvalidValuesForUniqueProperty( m, line );
- }
- else {
- //maybe put a byte buffer infront.
- RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
- HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_INDEX.getColumnFamily() )
- //not sure if I trust the lower two settings as it might iterfere with paging or set
- // arbitrary limits and what I want to retrieve.
- //That needs to be verified.
- .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
-
-
- RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
-
- while ( rangeSlicesIterator.hasNext() ) {
- Row rangeSliceValue = rangeSlicesIterator.next();
-
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
- ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
- //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
- // ones return garbage data.
- //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
- String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
- Charset.defaultCharset() ).trim();
+ ObjectMapper objectMapper = new ObjectMapper( );
+ File jsonObjectFile = new File(line.getOptionValue( "file" ));
- //defensive programming, don't have to have to parse the string if it doesn't contain users.
- if (returnedRowKey.contains("users") && returnedRowKey.contains( "email" ) ) {
+ JsonNode node = objectMapper.readTree( jsonObjectFile );
- String[] parsedRowKey = returnedRowKey.split( ":" );
+ JsonNode users = node.get( "user" );
- //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
- if ( parsedRowKey.length > 4 ) {
- parsedRowKey = garbageRowKeyParser( parsedRowKey );
+ for(JsonNode email:users) {
- if ( parsedRowKey == null ) {
- logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
- continue;
- }
- }
- //if the rowkey contains less than four parts then it is completely invalid
- else if ( parsedRowKey.length < 4 ) {
- logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
- continue;
- }
+ String extractedEmail = email.get( "name" ).getTextValue();
- UUID applicationId = null;
- try {
- applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
- }
- catch ( Exception e ) {
- logger.error( "could not parse {} despite earlier parsing. Skipping...", parsedRowKey[0] );
- continue;
- }
- String collectionName = parsedRowKey[1];
- String uniqueValueKey = parsedRowKey[2];
- String uniqueValue = parsedRowKey[3];
+ Query query = new Query();
+ query.setEntityType( "user" );
+ query.addEqualityFilter( "email", extractedEmail );
+ //maybe this could be changed to detect duplicates
+ query.setLimit( 1 );
+ query.setResultsLevel( REFS );
-
- if ( collectionName.equals( "users" ) ) {
-
- ColumnSlice<ByteBuffer, ByteBuffer> columnSlice=rangeSliceValue.getColumnSlice();
-
- //if ( columnSlice.getColumns().size() != 0 ) {
- List<HColumn<ByteBuffer, ByteBuffer>> cols=columnSlice.getColumns();
-
- entityStateLogger(applicationId ,uniqueValue, cols );
- }
+ Results r = emf.getEntityManager( MANAGEMENT_APPLICATION_ID ).getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) ).searchCollection( "users", query );
+ if ( r != null && r.getRef() != null ) {
+ if ( em.get( r.getRef().getUuid() ) == null ) {
+ logger.error( "Email: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail,
+ r.getRef().getUuid() );
}
- }
- }
- logger.info( "Completed logging successfully" );
- }
-
-
- //Returns a functioning rowkey if it can otherwise returns null
- public String[] garbageRowKeyParser( String[] parsedRowKey ) {
- String[] modifiedRowKey = parsedRowKey.clone();
- while ( modifiedRowKey != null ) {
- if ( modifiedRowKey.length < 4 ) {
- return null;
- }
-
- String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
- if ( recreatedRowKey == null ) {
- recreatedRowKey = "";
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- }
- else {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- break;
- }
- }
- return modifiedRowKey;
- }
-
-
- private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
- for ( int i = 1; i < modifiedRowKey.length; i++ ) {
-
- recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
- if ( i + 1 != modifiedRowKey.length ) {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- }
- }
- modifiedRowKey = recreatedRowKey.split( ":" );
- return modifiedRowKey;
- }
-
-
- private void entityStateLogger(final UUID entityUuid,final String uniqueValue, final List<HColumn<ByteBuffer, ByteBuffer>> cols ) throws Exception {
-
-// UserInfo userInfo = null;
-// EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- //IndexBucketScanner scanner = new IndexBucketScanner( )
-
- //HColumn<ByteBuffer, ByteBuffer> col2 = cols.get( 0 );
- DynamicComposite dynamicComposite = dce.fromByteBuffer( cols.get( 0 ).getName());
-
- Object[] loggerObject = new Object[4];
- loggerObject[0] = entityUuid;
- loggerObject[1] = (String) dynamicComposite.get( 1 );
- loggerObject[2] = (UUID) dynamicComposite.get( 2 );
- loggerObject[3] = new Date( UUIDUtils.getTimestampInMillis( (UUID) dynamicComposite.get( 3 ) ));
-
- logger.info("Found the following orguuid: {} associated with the email: {} that has the following uuid: {} and it was created at time: {} ",loggerObject);
- }
-
-
- private void orgVerification( final String uniqueValue, final EntityManager em, final EntityRef orgRef,
- final UUID uuid, final String s2 ) throws Exception {
- try {
- em.getDictionaryAsMap( orgRef, "orgProperties" );
- OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( uuid );
-
- if(organizationInfo == null) {
- logger.error( "The following email works: {} but the orgUUID: {} returns null.",uniqueValue,uuid );
- }
- else {
-
- Object[] loggingObject = new Object[3];
- loggingObject[0] = uniqueValue;
- loggingObject[1] = organizationInfo.getName();
- loggingObject[2] = organizationInfo.getUuid();
-
- logger.info( "The following email works: {} with the following orgname: {} and orgUUID: {}",
- loggingObject );
- }
- }
- catch ( EntityNotFoundException enfe ) {
- logger.error( s2, uniqueValue, uuid );
-
- BiMap<UUID,String> applicationBiMap = managementService.getApplicationsForOrganization( uuid );
- if(applicationBiMap==null || applicationBiMap.size() == 0){
- logger.error("Applications didn't return for orgUUID: {}",uuid);
- }
- else if (applicationBiMap.size()>0){
- for ( Map.Entry<UUID, String> app : applicationBiMap.entrySet() ) {
- Object[] loggingObject = new Object[3];
- loggingObject[0] = app.getKey();
- loggingObject[1] = app.getValue();
- loggingObject[2] = uuid;
- logger.info( "Application with uuid: {} and name: {} was returned from orgUUID: {} ",loggingObject );
-
+ else {
+ logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES", extractedEmail,
+ r.getRef().getUuid() );
}
}
-
- }
- }
-
-
- //really only deletes ones that aren't existant for a specific value
- private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
- UUID applicationId = MANAGEMENT_APPLICATION_ID;
- String collectionName = "users"; //line.getOptionValue( COLLECTION_ARG );
- String uniqueValueKey = "email"; //line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
- String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
-
- //PLEASE ADD VERIFICATION.
-
- Object key = key( applicationId, collectionName,"email", uniqueValue );
-
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols = cass.getColumns( cass.getApplicationKeyspace( applicationId), ENTITY_UNIQUE, key, null, null, 1000,
- false );
-
-
- if ( cols.size() == 0 ) {
- logger.error( "This row key: {} has zero columns", key.toString() );
- }
- //TODO: fix to run on single entity once you can decode the column and row key.
-
- // entityStateLogger( uniqueValue, cols );
- }
-
-
- private String uuidGarbageParser( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- logger.error( "{} is unparsable", garbageString );
- break;
+ else{
+ logger.error( "Email: {} doesn't exist in ENTITY_INDEX.",extractedEmail );
}
}
- return stringToBeTruncated;
- }
-
-
- private String uuidStringVerifier( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- return null;
- }
- }
- return stringToBeTruncated;
+ logger.info( "Completed logging successfully" );
}
-
}
[4/4] usergrid git commit: Added fixes to delete specific entry from
entity_index. Reverted most changes to audit tool so that it would still
audit and wouldn't touch anything. Added new tool that does the above.
Changed query processor and relation mana
Posted by gr...@apache.org.
Added fixes to delete specific entry from entity_index.
Reverted most changes to audit tool so that it would still audit and wouldn't touch anything.
Added new tool that does the above. Changed query processor and relation manager to expose methods that allow you to see specific column and buffer that stores that column.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9e526ba5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9e526ba5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9e526ba5
Branch: refs/heads/unique_index_logging
Commit: 9e526ba5b9801bd0cb7f63ed980b7b21a3c2269e
Parents: 7ce5675
Author: George Reyes <gr...@apache.org>
Authored: Sat Dec 12 11:39:18 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Sat Dec 12 11:39:18 2015 -0800
----------------------------------------------------------------------
.../persistence/cassandra/QueryProcessor.java | 37 +++
.../cassandra/RelationManagerImpl.java | 33 +-
.../query/ir/result/AbstractScanColumn.java | 5 +
.../usergrid/tools/ManagementUserAudit.java | 91 +-----
.../tools/ManagementUserIndexMissingFix.java | 305 +++++++++++++++++++
5 files changed, 385 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e526ba5/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
index 538f6d6..75c08dc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessor.java
@@ -314,6 +314,43 @@ public class QueryProcessor {
return results;
}
+ /**
+ * Return the iterator results, ordered if required
+ */
+ public List<ScanColumn> getRawColumnResults( final SearchVisitorFactory searchVisitorFactory ) throws Exception {
+ // if we have no order by just load the results
+
+ if ( rootNode == null ) {
+ return null;
+ }
+
+ //use the gather iterator to collect all the '
+ final int resultSetSize = Math.min( size, Query.MAX_LIMIT );
+
+ ResultIterator itr = new GatherIterator(resultSetSize, rootNode, searchVisitorFactory.createVisitors(), executorService.getExecutor() );
+
+ List<ScanColumn> entityIds = new ArrayList<ScanColumn>( );
+
+ CursorCache resultsCursor = new CursorCache();
+
+ while ( entityIds.size() < size && itr.hasNext() ) {
+ entityIds.addAll( itr.next() );
+ }
+
+ //set our cursor, we paged through more entities than we want to return
+ if ( entityIds.size() > 0 ) {
+ int resultSize = Math.min( entityIds.size(), size );
+ entityIds = entityIds.subList( 0, resultSize );
+
+ //set our cursor on the last results
+ if ( resultSize == size ) {
+ entityIds.get( resultSize -1 ).addToCursor( resultsCursor );
+ }
+ }
+
+ return entityIds;
+ }
+
private class TreeEvaluator implements QueryVisitor {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e526ba5/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index a44b530..b379525 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -58,6 +58,7 @@ import org.apache.usergrid.persistence.query.ir.result.CollectionSearchVisitorFa
import org.apache.usergrid.persistence.query.ir.result.ConnectionResultsLoaderFactory;
import org.apache.usergrid.persistence.query.ir.result.ConnectionSearchVisitorFactory;
import org.apache.usergrid.persistence.query.ir.result.ConnectionTypesIterator;
+import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.IndexUtils;
import org.apache.usergrid.utils.MapUtils;
@@ -1676,7 +1677,7 @@ public class RelationManagerImpl implements RelationManager {
}
em.delete( itemRef );
return;
- }
+ }
Entity itemEntity = em.get( itemRef );
@@ -1793,6 +1794,36 @@ public class RelationManagerImpl implements RelationManager {
}
+ public List<ScanColumn> searchRawCollection( String collectionName, Query query ) throws Exception {
+ if ( query == null ) {
+ query = new Query();
+ }
+
+ headEntity = em.validate( headEntity );
+
+ CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collectionName );
+
+ query.setEntityType( collection.getType() );
+
+ final CollectionResultsLoaderFactory factory = new CollectionResultsLoaderFactory();
+
+ // we have something to search with, visit our tree and evaluate the
+ // results
+ QueryProcessor qp = new QueryProcessor( em, executorService, query, collection, factory );
+
+ CollectionSearchVisitorFactory collectionSearchVisitorFactory =
+ new CollectionSearchVisitorFactory( cass, indexBucketLocator, qp, applicationId, headEntity,
+ collectionName );
+ // SearchCollectionVisitor visitor = new SearchCollectionVisitor( this, qp );
+
+ return qp.getRawColumnResults( collectionSearchVisitorFactory );
+ }
+
+ public EntityRef returnHeadEntity(){
+ return headEntity;
+ }
+
+
@Override
@Metered( group = "core", name = "RelationManager_createConnection_connection_ref" )
public ConnectionRef createConnection( ConnectionRef connection ) throws Exception {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e526ba5/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
index f99103c..5adb4fc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/query/ir/result/AbstractScanColumn.java
@@ -51,6 +51,10 @@ public abstract class AbstractScanColumn implements ScanColumn {
return uuid;
}
+ public ByteBuffer getByteBuffer(){
+ return buffer;
+ }
+
public ByteBuffer getCursorValue() {
@@ -58,6 +62,7 @@ public abstract class AbstractScanColumn implements ScanColumn {
}
+
@Override
public boolean equals( Object o ) {
if ( this == o ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e526ba5/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
index 3d06021..eeb2575 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
@@ -34,35 +34,12 @@ import org.apache.commons.cli.Options;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.RelationManager;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
-import org.apache.usergrid.persistence.cassandra.IndexUpdate;
-import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
-import org.apache.usergrid.persistence.schema.CollectionInfo;
-
-import me.prettyprint.cassandra.service.RangeSlicesIterator;
-import me.prettyprint.hector.api.Keyspace;
+
import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.beans.Row;
-import me.prettyprint.hector.api.factory.HFactory;
-import me.prettyprint.hector.api.mutation.Mutator;
-import me.prettyprint.hector.api.query.QueryResult;
-import me.prettyprint.hector.api.query.RangeSlicesQuery;
-import me.prettyprint.hector.api.query.SliceQuery;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.Results.Level.REFS;
-import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
+
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.dce;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
@@ -185,8 +162,6 @@ public class ManagementUserAudit extends ToolBase {
Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- //searchEntityIndex( em, extractedEmail, line );
-
List<HColumn<ByteBuffer, ByteBuffer>> cols =
cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, be, be );
@@ -202,7 +177,6 @@ public class ManagementUserAudit extends ToolBase {
else {
logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES for ENTITY_UNIQUE", extractedEmail,
uuid );
- searchEntityIndex( em, extractedEmail, line );
}
}
else {
@@ -217,72 +191,19 @@ public class ManagementUserAudit extends ToolBase {
uuid = ue.fromByteBuffer( col.getName() );
Entity entity = em.get( uuid );
if ( entity == null ) {
- logger.error( "Email: {} with duplicate uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail, uuid );
+ logger.error( "Email: {} with duplicate uuid: {} doesn't exist in ENTITY_PROPERTIES.",
+ extractedEmail, uuid );
}
else {
Object[] loggerObject = new Object[3];
loggerObject[0] = extractedEmail;
loggerObject[1] = uuid;
loggerObject[2] = entity;
- logger.info( "Email: {} with duplicate uuid: {} with the following data: {} exists in ENTITY_PROPERTIES", extractedEmail, uuid );
+ logger.info( "Email: {} with duplicate uuid: {} with the following data: {} exists in "
+ + "ENTITY_PROPERTIES", extractedEmail, uuid );
}
}
}
}
}
-
-
- private void searchEntityIndex( final EntityManager em, final String extractedEmail, CommandLine line )
- throws Exception {
-
- Keyspace ko = cass.getUsergridApplicationKeyspace();
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- Query query = new Query();
- query.setEntityType( "user" );
- query.addEqualityFilter( "email", extractedEmail );
- query.setLimit( 1 );
- query.setResultsLevel( REFS );
-
- //String bucketId = emf.getEntityManager( MANAGEMENT_APPLICATION_ID ).loca .getBucket( indexedEntity.getUuid
- // () );
- RelationManagerImpl relationManager =
- ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
-
- Results r = relationManager.searchCollection( "users", query );
- if ( r != null && r.getRef() != null ) {
- if ( em.get( r.getRef().getUuid() ) == null ) {
-
- String bucketId = ( ( EntityManagerImpl ) em ).getIndexBucketLocator().getBucket( r.getId() );
-
- Object rowKey = key( relationManager.returnHeadEntity().getUuid()+":users:email", bucketId );
-
- final List<HColumn<ByteBuffer, ByteBuffer>> results =
- cass.getColumns( cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID ),
- ENTITY_INDEX.getColumnFamily(), rowKey, null, null, 1, false );
- if(results.size()==1) {
- logger.error("Deleting the following row key: {} that contains this column: {} ",rowKey,
- dce.fromByteBuffer(results.get( 0 ).getName()).get( 1 ) );
- //addDeleteToMutator( m, ENTITY_INDEX, rowKey, results.get( 0 ).getName(), createTimestamp() );
-
- // m.execute();
- }
- else{
- if(results.size() > 1) {
- logger.error( "Too many columns for row key: {}, Implement searching logic for proper email.",rowKey );
- }
- else{
- logger.error("Zero columns for row key: {}",rowKey);
- }
- }
- }
- else {
- logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES", extractedEmail,
- r.getRef().getUuid() );
- }
- }
- else {
- logger.error( "Email: {} doesn't exist in ENTITY_INDEX.", extractedEmail );
- }
- }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e526ba5/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
new file mode 100644
index 0000000..2a6b00f
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
@@ -0,0 +1,305 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.UUID;
+
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.RelationManager;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.persistence.cassandra.IndexUpdate;
+import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
+import org.apache.usergrid.persistence.query.ir.result.SecondaryIndexSliceParser;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.DynamicComposite;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.MutationResult;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.QueryResult;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+import me.prettyprint.hector.api.query.SliceQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.Results.Level.REFS;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.dce;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+
+
+/**
+ * This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users then it
+ * deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row. If
+ * there exists more than one existing column then the one with the most recent timestamp wins and the other is
+ * deleted.
+ *
+ * If you want the run the tool on their cluster the following is what you need to do nohup java
+ * -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+ *
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value
+ * <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class ManagementUserIndexMissingFix extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( ManagementUserIndexMissingFix.class );
+
+ private static final String FILE_PATH = "file";
+
+ private static final String DUPLICATE_EMAIL = "dup";
+
+ private static final String ROW_KEY = "row";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+ Option file_path =
+ OptionBuilder.withArgName( FILE_PATH ).hasArg().isRequired( false ).withDescription( "file path" )
+ .create( FILE_PATH );
+ options.addOption( file_path );
+
+ Option duplicate_email = OptionBuilder.withArgName( DUPLICATE_EMAIL ).hasArg().isRequired( false )
+ .withDescription( "duplicate email to examine" )
+ .create( DUPLICATE_EMAIL );
+ options.addOption( duplicate_email );
+
+ Option row_key = OptionBuilder.withArgName( ROW_KEY ).hasArg().isRequired( false )
+ .withDescription( "row key to check against" ).create( ROW_KEY );
+ options.addOption( row_key );
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique checker" );
+
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+
+ if ( line.getOptionValue( ( "file" ) ) == null ) {
+ if ( line.getOptionValue( "dup" ) != null ) {
+ String extractedEmail = line.getOptionValue( "dup" );
+ column_verification( em, extractedEmail, line );
+ }
+ else {
+ logger.error( "Need to have -file or -dup not both and certainly not neither." );
+ }
+ }
+ else {
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ File jsonObjectFile = new File( line.getOptionValue( "file" ) );
+
+ JsonNode node = objectMapper.readTree( jsonObjectFile );
+
+ JsonNode users = node.get( "user" );
+
+ for ( JsonNode email : users ) {
+
+ String extractedEmail = email.get( "name" ).getTextValue();
+ column_verification( em, extractedEmail, line );
+ }
+ logger.info( "Completed logging successfully" );
+ }
+ }
+
+
+ private void column_verification( final EntityManager em, final String extractedEmail, CommandLine line )
+ throws Exception {
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "users";
+ String uniqueValueKey = "email";
+ String uniqueValue = extractedEmail;
+
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, be, be );
+
+ if ( cols.size() == 1 ) {
+ UUID uuid = null;
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ uuid = ue.fromByteBuffer( col.getName() );
+ }
+ if ( em.get( uuid ) == null ) {
+ logger.error( "Email: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail, uuid );
+ }
+ else {
+ logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES for ENTITY_UNIQUE", extractedEmail,
+ uuid );
+ searchEntityIndex( em, extractedEmail, line );
+ }
+ }
+ else {
+ if ( cols.size() == 0 ) {
+ logger.error( "Email: {} doesn't exist in ENTITY_UNIQUE.", extractedEmail );
+ }
+ else {
+ logger.error( "Email: {} has {} number of duplicate columns in ENTITY_UNIQUE", extractedEmail,
+ cols.size() );
+ UUID uuid = null;
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ uuid = ue.fromByteBuffer( col.getName() );
+ Entity entity = em.get( uuid );
+ if ( entity == null ) {
+ logger.error( "Email: {} with duplicate uuid: {} doesn't exist in ENTITY_PROPERTIES.",
+ extractedEmail, uuid );
+ }
+ else {
+ Object[] loggerObject = new Object[3];
+ loggerObject[0] = extractedEmail;
+ loggerObject[1] = uuid;
+ loggerObject[2] = entity;
+ logger.info( "Email: {} with duplicate uuid: {} with the following data: {} exists in "
+ + "ENTITY_PROPERTIES", extractedEmail, uuid );
+ }
+ }
+ }
+ }
+ }
+
+
+ private void searchEntityIndex( final EntityManager em, final String extractedEmail, CommandLine line )
+ throws Exception {
+
+ Keyspace ko = cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ Query query = new Query();
+ query.setEntityType( "user" );
+ query.addEqualityFilter( "email", extractedEmail );
+ query.setLimit( 1 );
+ query.setResultsLevel( REFS );
+
+ RelationManagerImpl relationManager =
+ ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+
+ Results r = relationManager.searchCollection( "users", query );
+ if ( r != null && r.getRef() != null ) {
+ if ( em.get( r.getRef().getUuid() ) == null ) {
+
+ logger.info( "Trying to remove uuid: {} from ENTITY_INDEX.", r.getRef().getUuid() );
+
+
+ List<ScanColumn> entityIds = relationManager.searchRawCollection( "users", query );
+
+ for ( ScanColumn entityId : entityIds ) {
+ SecondaryIndexSliceParser.SecondaryIndexColumn secondaryIndexColumn =
+ ( SecondaryIndexSliceParser.SecondaryIndexColumn ) entityId;
+
+ DynamicComposite columnName = dce.fromByteBuffer( secondaryIndexColumn.getByteBuffer() );
+ String bucketId =
+ ( ( EntityManagerImpl ) em ).getIndexBucketLocator().getBucket( r.getRef().getUuid() );
+ Object index_name = key( MANAGEMENT_APPLICATION_ID, "users", "email" );
+
+
+ Object index_key = key( index_name, bucketId );
+ logger.info( "Deleting the following rowkey: {} from ENTITY_INDEX.", index_key );
+ addDeleteToMutator( m, ENTITY_INDEX, index_key, columnName, createTimestamp() );
+
+ m.execute();
+ }
+
+ Results secondResults = relationManager.searchCollection( "users", query );
+ if ( secondResults != null && secondResults.getRef() != null ) {
+ if ( secondResults.getRef().getUuid().equals( r.getRef().getUuid() ) ) {
+ logger.error( "Removing uuid: {} from ENTITY_INDEX did not work. Email: {} still broken.",
+ r.getRef().getUuid(), extractedEmail );
+ }
+ }
+ else {
+ logger.info( "Delete of uuid: {} from ENTITY_INDEX worked. Email: {} should work.",
+ r.getRef().getUuid(), extractedEmail );
+ }
+ }
+ else {
+ logger.error( "Uuid: {} returns a valid entity for email: {} in ENTITY_INDEX.", r.getRef().getUuid(),
+ extractedEmail );
+ }
+ }
+
+ else {
+ logger.error( "Email: {} doesn't exist in ENTITY_INDEX.", extractedEmail );
+ }
+ }
+}
[3/4] usergrid git commit: Changed audit tool to do a search on the
entity index as well. Need to find a way to delete a specific column.
Posted by gr...@apache.org.
Changed audit tool to do a search on the entity index as well. Need to find a way to delete a specific column.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7ce56750
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7ce56750
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7ce56750
Branch: refs/heads/unique_index_logging
Commit: 7ce5675055ab54c403adfeb14224509846aff4af
Parents: cabd54a
Author: George Reyes <gr...@apache.org>
Authored: Fri Dec 11 14:15:17 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Dec 11 14:15:17 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/ManagementUserAudit.java | 128 ++++++++++++++++---
1 file changed, 107 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7ce56750/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
index 9c100d9..3d06021 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
@@ -35,13 +35,34 @@ import org.apache.commons.cli.Options;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.Query;
-
+import org.apache.usergrid.persistence.RelationManager;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.persistence.cassandra.IndexUpdate;
+import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.HColumn;
-
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.QueryResult;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+import me.prettyprint.hector.api.query.SliceQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static org.apache.usergrid.persistence.Results.Level.REFS;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.dce;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
@@ -78,6 +99,8 @@ public class ManagementUserAudit extends ToolBase {
private static final String DUPLICATE_EMAIL = "dup";
+ private static final String ROW_KEY = "row";
+
@Override
@SuppressWarnings( "static-access" )
@@ -93,15 +116,19 @@ public class ManagementUserAudit extends ToolBase {
options.addOption( hostOption );
Option file_path =
- OptionBuilder.withArgName( FILE_PATH ).hasArg().isRequired( false )
- .withDescription( "file path" ).create( FILE_PATH );
+ OptionBuilder.withArgName( FILE_PATH ).hasArg().isRequired( false ).withDescription( "file path" )
+ .create( FILE_PATH );
options.addOption( file_path );
- Option duplicate_email =
- OptionBuilder.withArgName( DUPLICATE_EMAIL ).hasArg().isRequired( false )
- .withDescription( "duplicate email to examine" ).create( DUPLICATE_EMAIL );
+ Option duplicate_email = OptionBuilder.withArgName( DUPLICATE_EMAIL ).hasArg().isRequired( false )
+ .withDescription( "duplicate email to examine" )
+ .create( DUPLICATE_EMAIL );
options.addOption( duplicate_email );
+ Option row_key = OptionBuilder.withArgName( ROW_KEY ).hasArg().isRequired( false )
+ .withDescription( "row key to check against" ).create( ROW_KEY );
+ options.addOption( row_key );
+
return options;
}
@@ -120,13 +147,13 @@ public class ManagementUserAudit extends ToolBase {
EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
- if(line.getOptionValue( ("file") ) == null) {
- if ( line.getOptionValue( "dup" )!=null){
+ if ( line.getOptionValue( ( "file" ) ) == null ) {
+ if ( line.getOptionValue( "dup" ) != null ) {
String extractedEmail = line.getOptionValue( "dup" );
- column_verification( em, extractedEmail );
- }
- else{
- logger.error("Need to have -file or -dup not both and certainly not neither.");
+ column_verification( em, extractedEmail, line );
+ }
+ else {
+ logger.error( "Need to have -file or -dup not both and certainly not neither." );
}
}
else {
@@ -141,14 +168,16 @@ public class ManagementUserAudit extends ToolBase {
for ( JsonNode email : users ) {
String extractedEmail = email.get( "name" ).getTextValue();
- column_verification( em, extractedEmail );
+ column_verification( em, extractedEmail, line );
}
logger.info( "Completed logging successfully" );
}
}
- private void column_verification( final EntityManager em, final String extractedEmail ) throws Exception {UUID
- applicationId = MANAGEMENT_APPLICATION_ID;
+
+ private void column_verification( final EntityManager em, final String extractedEmail, CommandLine line )
+ throws Exception {
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
String collectionName = "users";
String uniqueValueKey = "email";
String uniqueValue = extractedEmail;
@@ -156,6 +185,8 @@ public class ManagementUserAudit extends ToolBase {
Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ //searchEntityIndex( em, extractedEmail, line );
+
List<HColumn<ByteBuffer, ByteBuffer>> cols =
cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, be, be );
@@ -166,11 +197,12 @@ public class ManagementUserAudit extends ToolBase {
uuid = ue.fromByteBuffer( col.getName() );
}
if ( em.get( uuid ) == null ) {
- logger.error( "Email: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail,
- uuid );
+ logger.error( "Email: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail, uuid );
}
else {
- logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES", extractedEmail, uuid );
+ logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES for ENTITY_UNIQUE", extractedEmail,
+ uuid );
+ searchEntityIndex( em, extractedEmail, line );
}
}
else {
@@ -185,8 +217,7 @@ public class ManagementUserAudit extends ToolBase {
uuid = ue.fromByteBuffer( col.getName() );
Entity entity = em.get( uuid );
if ( entity == null ) {
- logger.error( "Email: {} with duplicate uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail,
- uuid );
+ logger.error( "Email: {} with duplicate uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail, uuid );
}
else {
Object[] loggerObject = new Object[3];
@@ -199,4 +230,59 @@ public class ManagementUserAudit extends ToolBase {
}
}
}
+
+
+ private void searchEntityIndex( final EntityManager em, final String extractedEmail, CommandLine line )
+ throws Exception {
+
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ Query query = new Query();
+ query.setEntityType( "user" );
+ query.addEqualityFilter( "email", extractedEmail );
+ query.setLimit( 1 );
+ query.setResultsLevel( REFS );
+
+ //String bucketId = emf.getEntityManager( MANAGEMENT_APPLICATION_ID ).loca .getBucket( indexedEntity.getUuid
+ // () );
+ RelationManagerImpl relationManager =
+ ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+
+ Results r = relationManager.searchCollection( "users", query );
+ if ( r != null && r.getRef() != null ) {
+ if ( em.get( r.getRef().getUuid() ) == null ) {
+
+ String bucketId = ( ( EntityManagerImpl ) em ).getIndexBucketLocator().getBucket( r.getId() );
+
+ Object rowKey = key( relationManager.returnHeadEntity().getUuid()+":users:email", bucketId );
+
+ final List<HColumn<ByteBuffer, ByteBuffer>> results =
+ cass.getColumns( cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID ),
+ ENTITY_INDEX.getColumnFamily(), rowKey, null, null, 1, false );
+ if(results.size()==1) {
+ logger.error("Deleting the following row key: {} that contains this column: {} ",rowKey,
+ dce.fromByteBuffer(results.get( 0 ).getName()).get( 1 ) );
+ //addDeleteToMutator( m, ENTITY_INDEX, rowKey, results.get( 0 ).getName(), createTimestamp() );
+
+ // m.execute();
+ }
+ else{
+ if(results.size() > 1) {
+ logger.error( "Too many columns for row key: {}, Implement searching logic for proper email.",rowKey );
+ }
+ else{
+ logger.error("Zero columns for row key: {}",rowKey);
+ }
+ }
+ }
+ else {
+ logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES", extractedEmail,
+ r.getRef().getUuid() );
+ }
+ }
+ else {
+ logger.error( "Email: {} doesn't exist in ENTITY_INDEX.", extractedEmail );
+ }
+ }
}
[2/4] usergrid git commit: Added ability to check one email.
Posted by gr...@apache.org.
Added ability to check one email.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cabd54a1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cabd54a1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cabd54a1
Branch: refs/heads/unique_index_logging
Commit: cabd54a13c99aa25984be1bffc36df1fb87f381b
Parents: 5a96416
Author: George Reyes <gr...@apache.org>
Authored: Fri Dec 11 10:15:58 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Dec 11 10:15:58 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/ManagementUserAudit.java | 117 ++++++++++++-------
1 file changed, 74 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cabd54a1/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
index d1eea62..9c100d9 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
@@ -32,6 +32,7 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.Query;
@@ -73,7 +74,9 @@ public class ManagementUserAudit extends ToolBase {
private static final Logger logger = LoggerFactory.getLogger( ManagementUserAudit.class );
- private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "file";
+ private static final String FILE_PATH = "file";
+
+ private static final String DUPLICATE_EMAIL = "dup";
@Override
@@ -89,11 +92,15 @@ public class ManagementUserAudit extends ToolBase {
options.addOption( hostOption );
- Option entityUniquePropertyValue =
- OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( true )
- .withDescription( "file path" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
- options.addOption( entityUniquePropertyValue );
+ Option file_path =
+ OptionBuilder.withArgName( FILE_PATH ).hasArg().isRequired( false )
+ .withDescription( "file path" ).create( FILE_PATH );
+ options.addOption( file_path );
+ Option duplicate_email =
+ OptionBuilder.withArgName( DUPLICATE_EMAIL ).hasArg().isRequired( false )
+ .withDescription( "duplicate email to examine" ).create( DUPLICATE_EMAIL );
+ options.addOption( duplicate_email );
return options;
}
@@ -113,59 +120,83 @@ public class ManagementUserAudit extends ToolBase {
EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
- ObjectMapper objectMapper = new ObjectMapper( );
-
- File jsonObjectFile = new File(line.getOptionValue( "file" ));
-
- JsonNode node = objectMapper.readTree( jsonObjectFile );
-
- JsonNode users = node.get( "user" );
+ if(line.getOptionValue( ("file") ) == null) {
+ if ( line.getOptionValue( "dup" )!=null){
+ String extractedEmail = line.getOptionValue( "dup" );
+ column_verification( em, extractedEmail );
+ }
+ else{
+ logger.error("Need to have -file or -dup not both and certainly not neither.");
+ }
+ }
+ else {
+ ObjectMapper objectMapper = new ObjectMapper();
- for(JsonNode email:users) {
+ File jsonObjectFile = new File( line.getOptionValue( "file" ) );
- String extractedEmail = email.get( "name" ).getTextValue();
+ JsonNode node = objectMapper.readTree( jsonObjectFile );
- Query query = new Query();
- query.setEntityType( "user" );
- query.addEqualityFilter( "email", extractedEmail );
- //maybe this could be changed to detect duplicates
- query.setLimit( 1 );
- query.setResultsLevel( REFS );
+ JsonNode users = node.get( "user" );
+ for ( JsonNode email : users ) {
+ String extractedEmail = email.get( "name" ).getTextValue();
+ column_verification( em, extractedEmail );
+ }
+ logger.info( "Completed logging successfully" );
+ }
+ }
- UUID applicationId = MANAGEMENT_APPLICATION_ID;
- String collectionName = "users";
- String uniqueValueKey = "email";
- String uniqueValue = extractedEmail;
+ private void column_verification( final EntityManager em, final String extractedEmail ) throws Exception {UUID
+ applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "users";
+ String uniqueValueKey = "email";
+ String uniqueValue = extractedEmail;
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- List<HColumn<ByteBuffer, ByteBuffer>> cols = cass.getAllColumns( cass.getApplicationKeyspace( applicationId ),ENTITY_UNIQUE,key,be,be );
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, be, be );
- if ( cols.size() == 1 ) {
+ if ( cols.size() == 1 ) {
+ UUID uuid = null;
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ uuid = ue.fromByteBuffer( col.getName() );
+ }
+ if ( em.get( uuid ) == null ) {
+ logger.error( "Email: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail,
+ uuid );
+ }
+ else {
+ logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES", extractedEmail, uuid );
+ }
+ }
+ else {
+ if ( cols.size() == 0 ) {
+ logger.error( "Email: {} doesn't exist in ENTITY_UNIQUE.", extractedEmail );
+ }
+ else {
+ logger.error( "Email: {} has {} number of duplicate columns in ENTITY_UNIQUE", extractedEmail,
+ cols.size() );
UUID uuid = null;
for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- uuid = ue.fromByteBuffer( col.getName());
- }
- if ( em.get( uuid ) == null ) {
- logger.error( "Email: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail,uuid);
- }
- else {
- logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES", extractedEmail,uuid);
- }
- }
- else{
- if(cols.size() == 0) {
- logger.error( "Email: {} doesn't exist in ENTITY_UNIQUE.", extractedEmail );
- }
- else{
- logger.error("Email: {} has {} number of duplicate columns in ENTITY_UNIQUE",extractedEmail,cols.size());
+ uuid = ue.fromByteBuffer( col.getName() );
+ Entity entity = em.get( uuid );
+ if ( entity == null ) {
+ logger.error( "Email: {} with duplicate uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail,
+ uuid );
+ }
+ else {
+ Object[] loggerObject = new Object[3];
+ loggerObject[0] = extractedEmail;
+ loggerObject[1] = uuid;
+ loggerObject[2] = entity;
+ logger.info( "Email: {} with duplicate uuid: {} with the following data: {} exists in ENTITY_PROPERTIES", extractedEmail, uuid );
+ }
}
}
}
- logger.info( "Completed logging successfully" );
}
}