You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/12/18 20:35:16 UTC
[2/4] usergrid git commit: Updates to tools making them current. And
added new tool to look through entity index and figure out which
organizations are not in the unique index, thus repairing them.
Updates to tools making them current. And added new tool to look through entity index and figure out which organizations are not in the unique index, thus repairing them.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d2a9fb89
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d2a9fb89
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d2a9fb89
Branch: refs/heads/unique_index_logging
Commit: d2a9fb89c2af99ed4afb0eb67d0660b88e294781
Parents: 9ebbc1b
Author: George Reyes <gr...@apache.org>
Authored: Thu Dec 17 11:09:28 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Dec 17 11:09:28 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/ApplicationAuditOrgFix.java | 572 +++++++++++++++++++
.../usergrid/tools/ManagementUserAudit.java | 118 +++-
.../tools/ManagementUserIndexMissingFix.java | 2 +-
.../usergrid/tools/ManagementUserOrgAudit.java | 394 +++++++++++++
.../org/apache/usergrid/tools/OrgFixTool.java | 133 +++++
.../usergrid/tools/OrgUniqueIndexScanner.java | 476 +++++++++++++++
.../tools/OrganizationIndexMissingFix.java | 319 +++++++++++
7 files changed, 1998 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
new file mode 100644
index 0000000..d99d28f
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
@@ -0,0 +1,572 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.thrift.TBaseHelper;
+
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.PagingResultsIterator;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
+import org.apache.usergrid.persistence.cassandra.Serializers;
+import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.collect.BiMap;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.DynamicComposite;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.dce;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.ue;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+
+
+/**
+ * This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users then it
+ * deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row. If
+ * there exists more than one existing column then the one with the most recent timestamp wins and the other is
+ * deleted.
+ *
+ * If you want the run the tool on their cluster the following is what you need to do nohup java
+ * -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+ *
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value
+ * <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class ApplicationAuditOrgFix extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( ApplicationAuditOrgFix.class );
+
+ private static final String APP_VALUE = "app";
+
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+ Option appOption =
+ OptionBuilder.withArgName( "app" ).hasArg().isRequired( false ).withDescription( "Application uuid" )
+ .create( "app" );
+
+ options.addOption( appOption );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique checker" );
+
+
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Keyspace keyspaceforOrgAppFix = cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID );
+ Mutator<ByteBuffer> repairMutator = createMutator( cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID ), be );
+ int index = 0;
+ HashSet<UUID> orgUUIDs = getOrgUUIDS();
+
+
+ for ( UUID orgUUID : orgUUIDs ) {
+ OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( orgUUID );
+
+ if ( organizationInfo == null ) {
+
+ List<HColumn<DynamicComposite, ByteBuffer>> cols =
+ cass.getAllColumns( keyspaceforOrgAppFix, ENTITY_COMPOSITE_DICTIONARIES,
+ key( orgUUID, "connected_entities", "owns" ), dce, be );
+ if ( cols.size() == 0 ) {
+ logger.error( "There are no applications associated with this org: {}", orgUUID );
+ }
+ else {
+ Boolean orgFixed = false;
+ for ( HColumn<DynamicComposite, ByteBuffer> col : cols ) {
+ DynamicComposite dynamicComposite = col.getName();
+ UUID applicationId = ( UUID ) dynamicComposite.get( 0 );
+ logger.info( "The following applications are associated with this org uuid: {}. Trying to repair based on working application.",orgUUID ,applicationId );
+ ApplicationInfo applicationInfo = managementService.getApplicationInfo( applicationId );
+ if(applicationInfo == null){
+ logger.info("The following application uuid: {} returns null for org uuid: {}",applicationId,orgUUID);
+ }
+ else{
+ logger.info("The following application with information: {}, will be used to repair org uuid: {}.",applicationInfo,orgUUID);
+ }
+
+ }
+ }
+ }
+ else {
+
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "groups";
+ String uniqueValueKey = "path";
+ String uniqueValue = organizationInfo.getName();
+
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key(applicationId, collectionName, uniqueValueKey,uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
+ false );
+
+ if(cols.size() == 0){
+ logger.info("Organization uuid: {} with name: {} doesn't exist in ENTITY_UNIQUE. Repairing",organizationInfo.getUuid(),organizationInfo.getName());
+ addInsertToMutator( repairMutator, ENTITY_UNIQUE, key, orgUUID, null, createTimestamp() );
+ repairMutator.execute();
+ OrganizationInfo verificationOrganizationInfo = managementService.getOrganizationByName( organizationInfo.getName() );
+
+ if(verificationOrganizationInfo==null){
+ logger.error("Repair failed for rowkey: {} and org uuid: {}",key,orgUUID);
+ }
+ else if(verificationOrganizationInfo.getUuid().equals( organizationInfo.getUuid())&& verificationOrganizationInfo.equals( organizationInfo.getName() )){
+ logger.info( "Org name: {} with uuid is fixed: {}",organizationInfo.getUuid(),organizationInfo.getUuid() );
+ }
+ }
+ else {
+ Boolean correctUUIDFlag = false;
+ UUID alternateUUID = null;
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID uuid = Serializers.ue.fromByteBuffer( col.getName() );
+ if(uuid.equals( orgUUID )){
+ correctUUIDFlag = true;
+ }
+ alternateUUID = uuid;
+ }
+ if(!correctUUIDFlag){
+ logger.error("Found the same name: {} but different uuid: {} from the entity index: {}",new Object[]{organizationInfo.getName(),alternateUUID,organizationInfo.getUuid()});
+ }
+ else {
+ logger.info( "orgUUID: {} works", orgUUID );
+ }
+ }
+ }
+ }
+ logger.info( "Completed logging successfully. Found this many organizations: {}", orgUUIDs.size() );
+
+
+ // RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ // HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily(
+ // ENTITY_COMPOSITE_DICTIONARIES.getColumnFamily() )
+ // //not sure if I trust the lower two settings as it might iterfere with
+ // paging or set
+ // // arbitrary limits and what I want to retrieve.
+ // //That needs to be verified.
+ // .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+ //
+ //
+ // RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery,
+ // null, null );
+ //
+ // while ( rangeSlicesIterator.hasNext() ) {
+ // Row rangeSliceValue = rangeSlicesIterator.next();
+ //
+ //
+ // ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ // //Cassandra client library returns ByteBuffers that are views on top of a larger
+ // byte[]. These
+ // // ones return garbage data.
+ // //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ // String returnedRowKey =
+ // new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining
+ // (), Charset
+ // .defaultCharset() ).trim();
+ // logger.info( "HEre is a valid rowkey: {} ", returnedRowKey );
+ //
+ // //defensive programming, don't have to have to parse the string if it doesn't
+ // contain users.
+ //// if ( returnedRowKey.contains( "connected_entities" ) && returnedRowKey.contains(
+ // "owns" ) ) {
+ //// logger.info( "HEre is a valid rowkey: {} ", returnedRowKey );
+ //// }
+ // }
+
+
+ }
+
+ // //maybe put a byte buffer infront.
+ // RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ // HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily(
+ // ENTITY_COMPOSITE_DICTIONARIES.getColumnFamily() )
+ // //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // // arbitrary limits and what I want to retrieve.
+ // //That needs to be verified.
+ // .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+ //
+ //
+ // RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+ //
+ // while ( rangeSlicesIterator.hasNext() ) {
+ // Row rangeSliceValue = rangeSlicesIterator.next();
+ //
+ //
+ // ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ // //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These
+ // larger
+ // // ones return garbage data.
+ // //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ // String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf
+ // .remaining(),
+ // Charset.defaultCharset() ).trim();
+ //
+ // //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ // if (returnedRowKey.contains("connected_entities") && returnedRowKey.contains( "owns" )) {
+ //
+ // String[] parsedRowKey = returnedRowKey.split( ":" );
+ //
+ // if ( parsedRowKey.length < 3 ) {
+ // logger.error( "{} is a invalid row key, and unparseable due to length. Skipped...",
+ // returnedRowKey );
+ // continue;
+ // }
+ //
+ // parsedRowKey = garbageRowKeyParser( parsedRowKey );
+ //
+ // if ( parsedRowKey == null ) {
+ // logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ // continue;
+ // }
+ //
+ //
+ // orgUUIDs.add( UUID.fromString(parsedRowKey[0]) );
+ //
+ // ColumnSlice<ByteBuffer, ByteBuffer> columnSlice=rangeSliceValue.getColumnSlice();
+ // if ( columnSlice.getColumns().size() != 0 ) {
+ // List<HColumn<ByteBuffer, ByteBuffer>> cols=columnSlice.getColumns();
+ // for(HColumn<ByteBuffer,ByteBuffer> col: cols) {
+ // logger.info( "This is row key: {} 's column name: {} ",returnedRowKey,col );
+ //
+ // OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( UUID
+ // .fromString(parsedRowKey[0]) );
+ //
+ // logger.error("Probably blows up: {}.",ue.fromByteBuffer(col.getName()));
+ //// if(organizationInfo == null){
+ //// logger.error("Cannot retreive organization. check to see if can be repaired
+ // by application.");
+ //// }
+ // }
+ //
+ // }
+ //
+ //
+ //
+ //
+ //
+
+ //
+ // //if the rowkey contains more than 4 parts then it may have some garbage appended to the
+ // front.
+ // if ( parsedRowKey.length > 4 ) {
+ // parsedRowKey = garbageRowKeyParser( parsedRowKey );
+ //
+ // if ( parsedRowKey == null ) {
+ // logger.error( "{} is a invalid row key, and unparseable. Skipped...",
+ // returnedRowKey );
+ // continue;
+ // }
+ // }
+ // //if the rowkey contains less than four parts then it is completely invalid
+ // else if ( parsedRowKey.length < 4 ) {
+ // logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ // continue;
+ // }
+ //
+ // UUID applicationId = null;
+ // try {
+ // applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ // }
+ // catch ( Exception e ) {
+ // logger.error( "could not parse {} despite earlier parsing. Skipping...",
+ // parsedRowKey[0] );
+ // continue;
+ // }
+ // String collectionName = parsedRowKey[1];
+ // String uniqueValueKey = parsedRowKey[2];
+ // String uniqueValue = parsedRowKey[3];
+
+
+ // if ( collectionName.equals( "users" ) ) {
+
+ //ColumnSlice<ByteBuffer, ByteBuffer> columnSlice=rangeSliceValue.getColumnSlice();
+ //if ( columnSlice.getColumns().size() != 0 ) {
+ // List<HColumn<ByteBuffer, ByteBuffer>> cols=columnSlice.getColumns();
+
+ //entityStateLogger( applicationId,uniqueValue);
+ //}
+ // }
+ // }
+ // }
+
+
+ private HashSet<UUID> getOrgUUIDS() throws Exception {
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+ RelationManagerImpl relationManager =
+ ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+ Query query = new Query();
+ query.setLimit( 1000 );
+ query.setResultsLevel( Results.Level.IDS );
+
+ HashSet<UUID> orgUUIDs = new HashSet();
+
+ Results results = relationManager.searchCollection( "groups", query );
+
+ PagingResultsIterator pagingResultsIterator = new PagingResultsIterator( results );
+ while ( pagingResultsIterator.hasNext() ) {
+ orgUUIDs.add( ( UUID ) pagingResultsIterator.next() );
+ }
+ logger.info( "Found {} number of orgs", orgUUIDs.size() );
+ return orgUUIDs;
+ }
+
+
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 3 ) {
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
+
+ private void entityStateLogger( final UUID applicationUUID, final String uniqueValue ) throws Exception {
+
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+
+ ApplicationInfo applicationInfo = managementService.getApplicationInfo( applicationUUID );
+
+ if ( applicationInfo == null ) {
+ if ( !applicationUUID.equals( UUID.fromString( "00000000-0000-0000-0000-000000000010" ) ) ) {
+
+ logger.error( "Application uuid: {} and name: {} has return null.", applicationUUID, uniqueValue );
+ }
+ }
+ else {
+ OrganizationInfo organizationInfo = managementService.getOrganizationForApplication( applicationUUID );
+ if ( organizationInfo == null ) {
+ if ( !applicationUUID.equals( UUID.fromString( "00000000-0000-0000-0000-000000000001" ) ) ) {
+
+ logger.error (
+ "Application uuid: {} with name: {} is lost and has no organizations associated with it.",
+ applicationUUID, applicationInfo.getName() );
+ }
+ }
+ else {
+ EntityRef orgRef = new SimpleEntityRef( "group", organizationInfo.getUuid() );
+ orgVerification (
+ applicationUUID, uniqueValue, em, orgRef, organizationInfo.getUuid(),
+ "Application with uuid: {} with name: {} is missing their organization: {}" );
+ }
+ }
+ }
+
+
+ private void orgVerification( final UUID applicationUUID, final String uniqueValue, final EntityManager em,
+ final EntityRef orgRef, final UUID uuid, final String s2 ) throws Exception {
+ try {
+ em.getDictionaryAsMap( orgRef, "orgProperties" );
+
+ Object[] loggingObject = new Object[3];
+ loggingObject[0] = applicationUUID;
+ loggingObject[1] = uniqueValue;
+ loggingObject[2] = uuid;
+ logger.info( "Application with uuid: {} and name: {} was returned from orgUUID: {} ", loggingObject );
+ }
+ catch ( EntityNotFoundException enfe ) {
+ Object[] notFoundLogger = new Object[3];
+ notFoundLogger[0] = applicationUUID;
+ notFoundLogger[1] = uniqueValue;
+ notFoundLogger[2] = uuid;
+
+ logger.error( s2, uniqueValue, uuid );
+
+ BiMap<UUID, String> applicationBiMap = managementService.getApplicationsForOrganization( uuid );
+ if ( applicationBiMap == null || applicationBiMap.size() == 0 ) {
+ logger.error( "Applications didn't return for orgUUID: {}", uuid );
+ }
+ else if ( applicationBiMap.size() > 0 ) {
+ for ( Map.Entry<UUID, String> app : applicationBiMap.entrySet() ) {
+ Object[] loggingObject = new Object[3];
+ loggingObject[0] = app.getKey();
+ loggingObject[1] = app.getValue();
+ loggingObject[2] = uuid;
+ logger.info ( "Application with uuid: {} and name: {} was returned from orgUUID: {} ",
+ loggingObject );
+ }
+ }
+ }
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
+ UUID applicationId = UUID.fromString( line.getOptionValue( APP_VALUE ) );
+ String collectionName = "applications"; //line.getOptionValue( COLLECTION_ARG );
+ String uniqueValueKey = "name"; //line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+
+ // List<HColumn<ByteBuffer, ByteBuffer>> cols = cass.getColumns( cass.getApplicationKeyspace(
+ // applicationId), ENTITY_UNIQUE, key, null, null, 1000,
+ // false );
+
+
+ // if ( cols.size() == 0 ) {
+ // logger.error( "This row key: {} has zero columns", key.toString() );
+ // }
+
+ entityStateLogger( applicationId, uniqueValue );
+ }
+
+
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ logger.error( "{} is unparsable", garbageString );
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
index eeb2575..d6ac6c5 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
@@ -20,6 +20,8 @@ package org.apache.usergrid.tools;
import java.io.File;
import java.nio.ByteBuffer;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import org.codehaus.jackson.JsonNode;
@@ -32,8 +34,13 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+
+import com.google.common.collect.BiMap;
import me.prettyprint.hector.api.beans.HColumn;
@@ -162,6 +169,8 @@ public class ManagementUserAudit extends ToolBase {
Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ EntityManagerImpl emi = ( EntityManagerImpl ) em;
+
List<HColumn<ByteBuffer, ByteBuffer>> cols =
cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, be, be );
@@ -184,26 +193,105 @@ public class ManagementUserAudit extends ToolBase {
logger.error( "Email: {} doesn't exist in ENTITY_UNIQUE.", extractedEmail );
}
else {
- logger.error( "Email: {} has {} number of duplicate columns in ENTITY_UNIQUE", extractedEmail,
- cols.size() );
+ logger.warn( "Email: {} has {} number of duplicate columns in ENTITY_UNIQUE.Attempting Repair.",
+ extractedEmail, cols.size() );
UUID uuid = null;
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- uuid = ue.fromByteBuffer( col.getName() );
- Entity entity = em.get( uuid );
- if ( entity == null ) {
- logger.error( "Email: {} with duplicate uuid: {} doesn't exist in ENTITY_PROPERTIES.",
- extractedEmail, uuid );
+
+ //Only handles the case if there are two duplicates, if there are more then expand the tool. None
+ // have been found so far.
+
+ //if(cols.size()==2){
+ uuid = ue.fromByteBuffer( cols.get( 0 ).getName() );
+ Entity entity = em.get( uuid );
+ Long created = entity.getCreated();
+ UserInfo adminUserInfo = managementService.getAdminUserByUuid( uuid );
+
+
+ for ( int index = 1; index < cols.size(); index++ ) {
+ UUID duplicateEmailUUID = ue.fromByteBuffer( cols.get( index ).getName() );
+ Entity duplicateEntity = em.get( duplicateEmailUUID );
+
+ //if the dup is newer than the first entry
+ if ( created < duplicateEntity.getCreated() ) {
+ BiMap<UUID, String> uuidStringOrgBiMap =
+ managementService.getOrganizationsForAdminUser( duplicateEmailUUID );
+ BiMap<String, UUID> stringUUIDBiMap = uuidStringOrgBiMap.inverse();
+ for ( String orgName : stringUUIDBiMap.keySet() ) {
+ logger.warn( "Adding admin user: {} to organization: {}", adminUserInfo, orgName );
+ OrganizationInfo organizationInfo = managementService
+ .getOrganizationByUuid( stringUUIDBiMap.get( orgName ) );
+ managementService.addAdminUserToOrganization
+ (adminUserInfo,organizationInfo , false );
+ }
+ logger.warn( "Deleting duplicated uuid: {}.", duplicateEmailUUID );
+ emi.deleteEntity( duplicateEmailUUID );
}
- else {
- Object[] loggerObject = new Object[3];
- loggerObject[0] = extractedEmail;
- loggerObject[1] = uuid;
- loggerObject[2] = entity;
- logger.info( "Email: {} with duplicate uuid: {} with the following data: {} exists in "
- + "ENTITY_PROPERTIES", extractedEmail, uuid );
+ else if ( created > duplicateEntity.getCreated() ) {
+ logger.info("older column was returned later from cassandra");
+ BiMap<UUID, String> uuidStringOrgBiMap = managementService.getOrganizationsForAdminUser( uuid );
+ BiMap<String, UUID> stringUUIDBiMap = uuidStringOrgBiMap.inverse();
+ adminUserInfo = managementService.getAdminUserByUuid( duplicateEmailUUID );
+
+
+ for ( String orgName : stringUUIDBiMap.keySet() ) {
+ logger.warn( "Adding admin user: {} to organization: {}", adminUserInfo, orgName );
+ OrganizationInfo organizationInfo = managementService.getOrganizationByUuid(
+ stringUUIDBiMap.get( orgName ) );
+ managementService.addAdminUserToOrganization(adminUserInfo,organizationInfo , false );
+ }
+ logger.warn( "Deleting duplicated uuid: {}.", uuid );
+ emi.deleteEntity( uuid );
+ created = duplicateEntity.getCreated();
+ uuid = duplicateEmailUUID;
}
}
+
+ // }
+
+ // for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ // uuid = ue.fromByteBuffer( col.getName() );
+ // //managementService.get
+ // entity = em.get( uuid );
+ // adminUserInfo = managementService.getAdminUserByUuid( uuid );
+ //
+ // Map<String, Object>
+ // userOrganizationData = managementService.getAdminUserOrganizationData(
+ // adminUserInfo, true );
+ //
+ //
+ // if ( entity == null ) {
+ // logger.error( "Email: {} with duplicate uuid: {} doesn't exist in
+ // ENTITY_PROPERTIES.",
+ // extractedEmail, uuid );
+ // }
+ // else {
+ //
+ //
+ //
+ //
+ // Object[] loggerObject = new Object[4];
+ // loggerObject[0] = extractedEmail;
+ // loggerObject[1] = uuid;
+ // loggerObject[2] = adminUserInfo;
+ // loggerObject[3] = userOrganizationData;
+ // logger.warn( "Email: {} with duplicate uuid: {} with the following data:
+ // {} and organizational data: {} exists in "
+ // + "ENTITY_PROPERTIES", loggerObject);
+ //
+
+ // Set<String> dictionaries = emi.getDictionaryNames( entity );
+ // if ( dictionaries != null ) {
+ // for ( String dictionary : dictionaries ) {
+ // Set<Object> values = emi.getDictionaryAsSet( entity, dictionary );
+ // logger.warn("The uuid: {} has the following dictionary: {} with the following values: {}.",new Object[] {uuid,dictionary,values});
+ // for ( Object value : values ) {
+ // emi.batchUpdateDictionary( m, entity, dictionary, value, true, timestampUuid );
+ // }
+ // }
+ // }
+ // }
}
}
}
}
+//}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
index 5d075fc..09e3202 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
@@ -286,7 +286,7 @@ public class ManagementUserIndexMissingFix extends ToolBase {
//preconditions should be satisfied before trying this method. Preconditions satisfy that
//uuid doesn't have valid data anywhere else but in INDEX for specific email. May still have the
//broken uuid else where but not of concern to users.
-
+
Object index_key = key( index_name, bucketId );
logger.info( "Deleting the following rowkey: {} from ENTITY_INDEX.", index_key );
addDeleteToMutator( m, ENTITY_INDEX, index_key, columnName, createTimestamp() );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserOrgAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserOrgAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserOrgAudit.java
new file mode 100644
index 0000000..8ef94fc
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserOrgAudit.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.thrift.TBaseHelper;
+
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.collect.BiMap;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+
+
+/**
+ * This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users then it
+ * deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row. If
+ * there exists more than one existing column then the one with the most recent timestamp wins and the other is
+ * deleted.
+ *
+ * If you want the run the tool on their cluster the following is what you need to do nohup java
+ * -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+ *
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value
+ * <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class ManagementUserOrgAudit extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( ManagementUserOrgAudit.class );
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique checker" );
+
+
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ if ( line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ deleteInvalidValuesForUniqueProperty( m, line );
+ }
+ else {
+ //maybe put a byte buffer infront.
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
+ //That needs to be verified.
+ .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+
+
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
+
+
+ ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
+ // ones return garbage data.
+ //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
+ Charset.defaultCharset() ).trim();
+
+
+ //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ if (returnedRowKey.contains("email") && returnedRowKey.contains( "users" ) && returnedRowKey.contains( MANAGEMENT_APPLICATION_ID.toString() )) {
+
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+
+ //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
+ if ( parsedRowKey.length > 4 ) {
+ parsedRowKey = garbageRowKeyParser( parsedRowKey );
+
+ if ( parsedRowKey == null ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ continue;
+ }
+ }
+ //if the rowkey contains less than four parts then it is completely invalid
+ else if ( parsedRowKey.length < 4 ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ continue;
+ }
+
+ UUID applicationId = null;
+ try {
+ applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ }
+ catch ( Exception e ) {
+ logger.error( "could not parse {} despite earlier parsing. Skipping...", parsedRowKey[0] );
+ continue;
+ }
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
+
+
+ if ( collectionName.equals( "users" ) ) {
+
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice=rangeSliceValue.getColumnSlice();
+ //if ( columnSlice.getColumns().size() != 0 ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> cols=columnSlice.getColumns();
+
+ entityStateLogger( uniqueValue, cols );
+ }
+ }
+ }
+ }
+ logger.info( "Completed logging successfully" );
+ }
+
+
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 4 ) {
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
+
+ private void entityStateLogger( final String uniqueValue, final List<HColumn<ByteBuffer, ByteBuffer>> cols ) throws Exception {
+
+ UserInfo userInfo = null;
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+
+ try {
+ userInfo = managementService.getAdminUserByEmail( uniqueValue );
+ }catch(Exception e){
+ logger.error("threw exception when looking up email: {}",uniqueValue);
+ e.printStackTrace();
+ }
+ if(userInfo==null) {
+ if(cols!=null){
+ if(cols.size()>1){
+ for(HColumn<ByteBuffer, ByteBuffer> col : cols) {
+ logger.warn( "This uuid: {} is associated with this duplicated email {}", ue.fromByteBuffer( col.getName()),uniqueValue );
+ }
+
+ }
+ if(cols.size()==1){
+ logger.error( "Management user with uuid: {} and email: {} is broken.",ue.fromByteBuffer( cols.get( 0 ).getName()), uniqueValue );
+ }
+ else{
+ logger.error( "Management user with email: {} is broken and has no uuid's associated with it",uniqueValue );
+ }
+ }
+ }
+ else {
+ Results orgResults =
+ em.getCollection( new SimpleEntityRef( User.ENTITY_TYPE, userInfo.getUuid() ), "groups", null, 10000, Results.Level.REFS,
+ false );
+
+ if(orgResults.getRefs().size() == 1){
+ EntityRef orgRef = orgResults.getRef();
+ orgVerification( uniqueValue, em, orgRef, orgRef.getUuid(),
+ "Management user with email: {} is present but cannot login due to missing their only organization: {}" );
+ }
+ else {
+ for ( EntityRef orgRef : orgResults.getRefs() ) {
+ orgVerification( uniqueValue, em, orgRef, orgRef.getUuid(),
+ "Management user with email: {} is present with multiple orgs but is missing the following orgUUID: {}" );
+
+ }
+ }
+ }
+
+ }
+
+
+ private void orgVerification( final String uniqueValue, final EntityManager em, final EntityRef orgRef,
+ final UUID uuid, final String s2 ) throws Exception {
+ try {
+ em.getDictionaryAsMap( orgRef, "orgProperties" );
+ OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( uuid );
+
+ if(organizationInfo == null) {
+ logger.error( "The following email works: {} but the orgUUID: {} returns null.",uniqueValue,uuid );
+ }
+ else {
+
+ Object[] loggingObject = new Object[3];
+ loggingObject[0] = uniqueValue;
+ loggingObject[1] = organizationInfo.getName();
+ loggingObject[2] = organizationInfo.getUuid();
+
+ logger.info( "The following email works: {} with the following orgname: {} and orgUUID: {}",
+ loggingObject );
+ }
+ }
+ catch ( EntityNotFoundException enfe ) {
+ logger.error( s2, uniqueValue, uuid );
+
+ BiMap<UUID,String> applicationBiMap = managementService.getApplicationsForOrganization( uuid );
+ if(applicationBiMap==null || applicationBiMap.size() == 0){
+ logger.error("Applications didn't return for orgUUID: {}",uuid);
+ }
+ else if (applicationBiMap.size()>0){
+ for ( Map.Entry<UUID, String> app : applicationBiMap.entrySet() ) {
+ Object[] loggingObject = new Object[3];
+ loggingObject[0] = app.getKey();
+ loggingObject[1] = app.getValue();
+ loggingObject[2] = uuid;
+ logger.info( "Application with uuid: {} and name: {} was returned from orgUUID: {} ",loggingObject );
+
+ }
+ }
+
+ }
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "users"; //line.getOptionValue( COLLECTION_ARG );
+ String uniqueValueKey = "email"; //line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key( applicationId, collectionName,"email", uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = cass.getColumns( cass.getApplicationKeyspace( applicationId), ENTITY_UNIQUE, key, null, null, 1000,
+ false );
+
+
+ if ( cols.size() == 0 ) {
+ logger.error( "This row key: {} has zero columns", key.toString() );
+ }
+
+ entityStateLogger( uniqueValue, cols );
+ }
+
+
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ logger.error( "{} is unparsable", garbageString );
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/OrgFixTool.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/OrgFixTool.java b/stack/tools/src/main/java/org/apache/usergrid/tools/OrgFixTool.java
new file mode 100644
index 0000000..e27aedc
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/OrgFixTool.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.util.List;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
+import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
+import org.apache.usergrid.persistence.query.ir.result.SecondaryIndexSliceParser;
+import org.apache.usergrid.persistence.query.ir.result.UUIDColumn;
+
+import me.prettyprint.hector.api.beans.DynamicComposite;
+
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.dce;
+
+
+/**
+ * Created by ApigeeCorporation on 12/15/15.
+ */
+public class OrgFixTool extends ToolBase {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( OrgFixTool.class );
+
+ private static final String FILE_PATH = "file";
+
+ private static final String DUPLICATE_EMAIL = "dup";
+
+ private static final String ROW_KEY = "row";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+
+ Option duplicate_email = OptionBuilder.withArgName( DUPLICATE_EMAIL ).hasArg().isRequired( false )
+ .withDescription( "duplicate org uuid to examine" )
+ .create( DUPLICATE_EMAIL );
+ options.addOption( duplicate_email );
+
+ return options;
+ }
+
+
+ @Override
+ public void runTool( final CommandLine line ) throws Exception {
+ startSpring();
+
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID);
+ RelationManagerImpl relationManager =
+ ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+
+ Query query = new Query();
+ query.setEntityType( "group" );
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Results.Level.REFS );
+
+ if(line.getOptionValue( "dup" )!=null) {
+ query.addEqualityFilter( "uuid",line.getOptionValue( "dup" ) );
+
+ logger.info( "This is what returns from the call to managements: {}",
+ managementService.getApplicationsForOrganization( UUID.fromString( line.getOptionValue( "dup" ) ) ) );
+ //managementService.getOrganizationByIdentifier( )
+
+ }
+ else {
+ List<ScanColumn> scanColumnList = relationManager.searchRawCollection( "groups", query ); //em.searchCollection( em.getApplicationRef(), "groups", query );
+
+ for ( ScanColumn scanColumn : scanColumnList ) {
+ logger.info( "This is what you get back: {}", scanColumn );
+ // SecondaryIndexSliceParser.SecondaryIndexColumn secondaryIndexColumn =
+ // ( SecondaryIndexSliceParser.SecondaryIndexColumn ) scanColumn;
+ UUIDColumn uuidColumn = ( UUIDColumn ) scanColumn;
+
+ //byte buffer from the scan column is the column name needed to delete from ENTITY_INDEX
+ //DynamicComposite columnName = dce.fromByteBuffer( uuidColumn.getByteBuffer() );
+
+
+ //logger.info("This is what you get back from the buffer: {}, from a buffer of size: {}",sc);
+
+ logger.info( "This is what returns from the call to managements: {}",
+ managementService.getApplicationsForOrganization( uuidColumn.getUUID() ) );
+
+
+ // if(em.get((UUID ) columnName.get( 2 ) ) == null){
+ // logger.error( "This organization is broken and doesn't appear in entity Properties: {}",columnName );
+ // }
+ // else{
+ // logger.info("This organization works: {}",columnName);
+ // }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/OrgUniqueIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/OrgUniqueIndexScanner.java b/stack/tools/src/main/java/org/apache/usergrid/tools/OrgUniqueIndexScanner.java
new file mode 100644
index 0000000..8c6fb33
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/OrgUniqueIndexScanner.java
@@ -0,0 +1,476 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.thrift.TimedOutException;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.thrift.TBaseHelper;
+
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.PagingResultsIterator;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
+import org.apache.usergrid.persistence.entities.Group;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
+import org.apache.usergrid.persistence.query.ir.result.UUIDColumn;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+
+
+/**
+ * This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users then it
+ * deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row. If
+ * there exists more than one existing column then the one with the most recent timestamp wins and the other is
+ * deleted.
+ *
+ * If you want the run the tool on their cluster the following is what you need to do nohup java
+ * -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+ *
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value
+ * <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class OrgUniqueIndexScanner extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( OrgUniqueIndexScanner.class );
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique index cleanup" );
+
+
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+// EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+//
+// RelationManagerImpl relationManager =
+// ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+//
+// Query query = new Query();
+// query.setEntityType( "group" );
+// query.setLimit( MAX_ENTITY_FETCH );
+// query.setResultsLevel( Results.Level.REFS );
+//
+// Results r = relationManager.searchCollection( "groups",query );
+
+ HashSet<UUID> orgUUIDS = getOrgUUIDS();
+ int index = 0;
+
+
+ if ( line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ deleteInvalidValuesForUniqueProperty( m, line,orgUUIDS );
+ }
+ else {
+
+
+ //maybe put a byte buffer infront.
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
+ //That needs to be verified.
+ .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+
+
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
+
+
+ ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
+ // ones return garbage data.
+ //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
+ Charset.defaultCharset() ).trim();
+
+
+ //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ if ( returnedRowKey.contains( "groups" ) && returnedRowKey
+ .contains( MANAGEMENT_APPLICATION_ID.toString() ) ) {
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+
+ //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
+ if ( parsedRowKey.length > 4 ) {
+ parsedRowKey = garbageRowKeyParser( parsedRowKey );
+
+ if ( parsedRowKey == null ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ continue;
+ }
+ }
+ //if the rowkey contains less than four parts then it is completely invalid
+ else if ( parsedRowKey.length < 4 ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ continue;
+ }
+
+ UUID applicationId = null;
+ try {
+ applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ }
+ catch ( Exception e ) {
+ logger.error( "could not parse {} despite earlier parsing. Skipping...", parsedRowKey[0] );
+ continue;
+ }
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
+
+
+ //if ( collectionName.equals( "users" ) ) {
+
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ //if ( columnSlice.getColumns().size() != 0 ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+
+
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,
+ returnedRowKey,orgUUIDS);
+ index++;
+ // }
+ // }
+ //}
+ }
+ }
+ }
+
+
+ logger.info("Completed repair successfully.Scanned over {} number of orgs",index);
+}
+
+
+
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 4 ) {
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
+
+ private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue ) throws Exception {
+ logger.debug( "Found 0 uuid's associated with {} Deleting row.", uniqueValue );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, timestamp );
+ mutator.execute();
+ return;
+ }
+
+
+ private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue,
+ final List<HColumn<ByteBuffer, ByteBuffer>> cols, String rowKey,HashSet<UUID> orgUUIDS) throws Exception {
+ Boolean cleanup = false;
+ EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+
+ int numberOfColumnsDeleted = 0;
+ //these columns all come from the same row key, which means they each belong to the same row key identifier
+ //thus mixing and matching them in the below if cases won't matter.
+ Entity[] entities = new Entity[cols.size()];
+ int numberOfRetrys = 8;
+ int numberOfTimesRetrying = 0;
+
+ int index = 0;
+
+
+ if ( cols.size() == 1 ) {
+ UUID uuid = null;
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ uuid = ue.fromByteBuffer( col.getName() );
+ }
+
+ if ( em.get( uuid ) == null ) {
+ logger.error( "OrgName: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", uniqueValue, uuid );
+
+ if(orgUUIDS.contains( uuid )){
+ logger.error( "OrgName: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES but exists in ENTITY_INDEX.Repairing...", uniqueValue, uuid );
+
+ Group organizationEntity = new Group();
+ organizationEntity.setPath( uniqueValue );
+ try {
+ em.create( uuid, Group.ENTITY_TYPE, organizationEntity.getProperties() );
+ organizationEntity = em.get( uuid, Group.class );
+ if ( organizationEntity == null ) {
+ logger.error( "OrgName: {} with uuid: {} could not be created", uniqueValue, uuid );
+ }
+ else{
+// logger.info( "OrgName: {} with uuid: {} was fixed.", uniqueValue,
+// uuid );
+ }
+ }catch(Exception e){
+ logger.error("Could not create: {} because of {}",uniqueValue,e.getMessage());
+ }
+
+ }
+ else{
+ logger.error( "OrgName: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES OR ENTITY INDEX. Leaving alone.", uniqueValue, uuid );
+
+ }
+ }
+ else {
+// logger.info( "OrgName: {} with uuid: {} exists in ENTITY_PROPERTIES for ENTITY_UNIQUE.", uniqueValue,
+// uuid );
+ }
+ }
+
+ }
+
+
+ private HashSet<UUID> getOrgUUIDS() throws Exception {
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+ RelationManagerImpl relationManager =
+ ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+ Query query = new Query();
+ query.setLimit( 1000 );
+ query.setResultsLevel( Results.Level.IDS );
+
+ HashSet<UUID> orgUUIDs = new HashSet( );
+
+ Results results = relationManager.searchCollection( "groups",query );
+
+ PagingResultsIterator pagingResultsIterator = new PagingResultsIterator( results );
+ while(pagingResultsIterator.hasNext()){
+ orgUUIDs.add((UUID) pagingResultsIterator.next());
+ }
+ logger.info( "Found {} number of orgs",orgUUIDs.size() );
+ return orgUUIDs;
+ }
+
+
+ private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
+ Entity entity = unverifiedEntity;
+ if ( entity != null && entity.getModified() == null ) {
+ if ( entity.getCreated() != null ) {
+ logger.debug( "{} has no modified. Subsituting created timestamp for their modified timestamp.Manually "
+ + "adding one for comparison purposes", entity.getUuid() );
+ entity.setModified( entity.getCreated() );
+ return entity;
+ }
+ else {
+ logger.error( "Found no created or modified timestamp. Please remake the following entity: {}."
+ + " Setting both created and modified to 1", entity.getUuid().toString() );
+ entity.setCreated( 1L );
+ entity.setModified( 1L );
+ return entity;
+ }
+ }
+ return entity;
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line,
+ final HashSet<UUID> orgUUIDS ) throws Exception {
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "groups";
+ String uniqueValueKey = "path";
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE);
+
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key(applicationId, collectionName, uniqueValueKey,uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
+ false );
+
+
+ if ( cols.size() == 0 ) {
+ logger.error( "This row key: {} has zero columns. Leaving alone.", key.toString() );
+ }
+ else if(cols.size() > 1){
+ logger.error( "This row key: {} has duplicate entries columns. Leaving alone.", key.toString() );
+
+ }
+
+// EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
+//
+// RelationManagerImpl relationManager =
+// ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+
+// Query query = new Query();
+// query.setEntityType( "group" );
+// query.setLimit( MAX_ENTITY_FETCH );
+// query.setResultsLevel( Results.Level.REFS );
+//
+// Results r = relationManager.searchCollection( "groups",query );
+//
+// managementService.getOrganizationByUuid( )
+//
+// logger.info("Does this work?: {}",r);
+
+
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols, key.toString(),orgUUIDS);
+ }
+
+
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ logger.error( "{} is unparsable", garbageString );
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+}