You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/12/18 20:35:15 UTC
[1/4] usergrid git commit: Updates to tools making them current. And
added new tool to look through entity index and figure out which
organizations are not in the unique index, thus repairing them.
Repository: usergrid
Updated Branches:
refs/heads/unique_index_logging 9ebbc1b90 -> 2627dbbfe
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationIndexMissingFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationIndexMissingFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationIndexMissingFix.java
new file mode 100644
index 0000000..b30d822
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationIndexMissingFix.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.UUID;
+
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.RelationManager;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.persistence.cassandra.IndexUpdate;
+import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
+import org.apache.usergrid.persistence.query.ir.result.SecondaryIndexSliceParser;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.DynamicComposite;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.MutationResult;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.QueryResult;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+import me.prettyprint.hector.api.query.SliceQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.Results.Level.REFS;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.dce;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+
+
+/**
+ * This utility audits all values in the ENTITY_UNIQUE column family. The way it does this is by taking in a *.json file
+ * that has an array of users then checking them to see if they are in the unique column family. Logging as it goes along.
+ *
+ * An additional use of this tool is to use the -dup flag like so
+ * java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar ManagementUserAudit -host localhost -dup grey@apache.org
+ *
+ * If we find a value that is present and also exists in ENTITY_PROPERTIES then it goes and queries the entity index.
+ * The reason this check has to be done is that it can be possible for an entity to have an invalid uuid in the entity index
+ * but has the correct data in ENTITY_UNIQUE. Causing a user to be unable to login.
+ * If they are in the index then we see if the uuid's that are stored in the index correspond to any entity in the ENTITY_PROPERTIES
+ * column family. If it does not then we retreive the column name from the RAW query then delete that column name from the bucket
+ * that contains the data. Thus freeing up the uuid to be queried by user name.
+ *
+ * THIS FIX SHOULD ONLY BE APPLIED IF PROBLEMS MATCH THESE OTHER CONDITIONS:
+ *
+ * When running the audit tools if you see two different uuid's for the same entity AND you cannot log in with email.
+ * If the unique audit found that this email should be working but the entity_index audit found it to be broken.
+ * IF you can PUT on the entity_unique uuid to update properties but CANNOT put on entity_index uuid.
+ *
+ * The above three conditions should be satisfied before this is ran on ANY user.
+ *
+ * @author grey
+ */
+public class OrganizationIndexMissingFix extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( OrganizationIndexMissingFix.class );
+
+ private static final String FILE_PATH = "file";
+
+ private static final String DUPLICATE_EMAIL = "dup";
+
+ private static final String ROW_KEY = "row";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+ Option file_path =
+ OptionBuilder.withArgName( FILE_PATH ).hasArg().isRequired( false ).withDescription( "file path" )
+ .create( FILE_PATH );
+ options.addOption( file_path );
+
+ Option duplicate_email = OptionBuilder.withArgName( DUPLICATE_EMAIL ).hasArg().isRequired( false )
+ .withDescription( "duplicate email to examine" )
+ .create( DUPLICATE_EMAIL );
+ options.addOption( duplicate_email );
+
+ Option row_key = OptionBuilder.withArgName( ROW_KEY ).hasArg().isRequired( false )
+ .withDescription( "row key to check against" ).create( ROW_KEY );
+ options.addOption( row_key );
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique checker" );
+
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+
+ if ( line.getOptionValue( ( "file" ) ) == null ) {
+ if ( line.getOptionValue( "dup" ) != null ) {
+ String extractedEmail = line.getOptionValue( "dup" );
+ column_verification( em, extractedEmail, line );
+ }
+ else {
+ logger.error( "Need to have -file or -dup not both and certainly not neither." );
+ }
+ }
+ else {
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ File jsonObjectFile = new File( line.getOptionValue( "file" ) );
+
+ JsonNode node = objectMapper.readTree( jsonObjectFile );
+
+ JsonNode users = node.get( "user" );
+
+ for ( JsonNode email : users ) {
+
+ String extractedEmail = email.get( "name" ).getTextValue();
+ column_verification( em, extractedEmail, line );
+ }
+ logger.info( "Completed logging successfully" );
+ }
+ }
+
+
+ private void column_verification( final EntityManager em, final String extractedEmail, CommandLine line )
+ throws Exception {
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "users";
+ String uniqueValueKey = "email";
+ String uniqueValue = extractedEmail;
+
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, be, be );
+
+ if ( cols.size() == 1 ) {
+ UUID uuid = null;
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ uuid = ue.fromByteBuffer( col.getName() );
+ }
+ if ( em.get( uuid ) == null ) {
+ logger.error( "Email: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", extractedEmail, uuid );
+ }
+ else {
+ logger.info( "Email: {} with uuid: {} exists in ENTITY_PROPERTIES for ENTITY_UNIQUE", extractedEmail,
+ uuid );
+ searchEntityIndex( em, extractedEmail, line );
+ }
+ }
+ else {
+ if ( cols.size() == 0 ) {
+ logger.error( "Email: {} doesn't exist in ENTITY_UNIQUE.", extractedEmail );
+ }
+ else {
+ logger.error( "Email: {} has {} number of duplicate columns in ENTITY_UNIQUE", extractedEmail,
+ cols.size() );
+ UUID uuid = null;
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ uuid = ue.fromByteBuffer( col.getName() );
+ Entity entity = em.get( uuid );
+ if ( entity == null ) {
+ logger.error( "Email: {} with duplicate uuid: {} doesn't exist in ENTITY_PROPERTIES.",
+ extractedEmail, uuid );
+ }
+ else {
+ Object[] loggerObject = new Object[3];
+ loggerObject[0] = extractedEmail;
+ loggerObject[1] = uuid;
+ loggerObject[2] = entity;
+ logger.info( "Email: {} with duplicate uuid: {} with the following data: {} exists in "
+ + "ENTITY_PROPERTIES", extractedEmail, uuid );
+ }
+ }
+ }
+ }
+ }
+
+
+ private void searchEntityIndex( final EntityManager em, final String extractedEmail, CommandLine line )
+ throws Exception {
+
+ Keyspace ko = cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ Query query = new Query();
+ query.setEntityType( "user" );
+ query.addEqualityFilter( "email", extractedEmail );
+ query.setLimit( 1 );
+ query.setResultsLevel( REFS );
+
+ RelationManagerImpl relationManager =
+ ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+
+ Results r = relationManager.searchCollection( "users", query );
+ if ( r != null && r.getRef() != null ) {
+ if ( em.get( r.getRef().getUuid() ) == null ) {
+
+ logger.info( "Trying to remove uuid: {} from ENTITY_INDEX.", r.getRef().getUuid() );
+
+
+ //This method didn't exist before and requires exposing the buffer with the ScanColumn information
+ //in the query processor. Hacky, but better than other methods.
+ List<ScanColumn> entityIds = relationManager.searchRawCollection( "users", query );
+
+ for ( ScanColumn entityId : entityIds ) {
+ SecondaryIndexSliceParser.SecondaryIndexColumn secondaryIndexColumn =
+ ( SecondaryIndexSliceParser.SecondaryIndexColumn ) entityId;
+
+ //byte buffer from the scan column is the column name needed to delete from ENTITY_INDEX
+ DynamicComposite columnName = dce.fromByteBuffer( secondaryIndexColumn.getByteBuffer() );
+ String bucketId =
+ ( ( EntityManagerImpl ) em ).getIndexBucketLocator().getBucket( r.getRef().getUuid() );
+ Object index_name = key( MANAGEMENT_APPLICATION_ID, "users", "email" );
+
+
+ //only delete from entity_index. Other data might contain valid references which is why
+ //preconditions should be satisfied before trying this method. Preconditions satisfy that
+ //uuid doesn't have valid data anywhere else but in INDEX for specific email. May still have the
+ //broken uuid else where but not of concern to users.
+
+ Object index_key = key( index_name, bucketId );
+ logger.info( "Deleting the following rowkey: {} from ENTITY_INDEX.", index_key );
+ addDeleteToMutator( m, ENTITY_INDEX, index_key, columnName, createTimestamp() );
+
+ m.execute();
+ }
+
+ Results secondResults = relationManager.searchCollection( "users", query );
+ if ( secondResults != null && secondResults.getRef() != null ) {
+ if ( secondResults.getRef().getUuid().equals( r.getRef().getUuid() ) ) {
+ logger.error( "Removing uuid: {} from ENTITY_INDEX did not work. Email: {} still broken.",
+ r.getRef().getUuid(), extractedEmail );
+ }
+ }
+ else {
+ logger.info( "Delete of uuid: {} from ENTITY_INDEX worked. Email: {} should work.",
+ r.getRef().getUuid(), extractedEmail );
+ }
+ }
+ else {
+ logger.error( "Uuid: {} returns a valid entity for email: {} in ENTITY_INDEX.", r.getRef().getUuid(),
+ extractedEmail );
+ }
+ }
+
+ else {
+ logger.error( "Email: {} doesn't exist in ENTITY_INDEX.", extractedEmail );
+ }
+ }
+}
[2/4] usergrid git commit: Updates to tools making them current. And
added new tool to look through entity index and figure out which
organizations are not in the unique index, thus repairing them.
Posted by gr...@apache.org.
Updates to tools making them current. And added new tool to look through entity index and figure out which organizations are not in the unique index, thus repairing them.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d2a9fb89
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d2a9fb89
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d2a9fb89
Branch: refs/heads/unique_index_logging
Commit: d2a9fb89c2af99ed4afb0eb67d0660b88e294781
Parents: 9ebbc1b
Author: George Reyes <gr...@apache.org>
Authored: Thu Dec 17 11:09:28 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Dec 17 11:09:28 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/ApplicationAuditOrgFix.java | 572 +++++++++++++++++++
.../usergrid/tools/ManagementUserAudit.java | 118 +++-
.../tools/ManagementUserIndexMissingFix.java | 2 +-
.../usergrid/tools/ManagementUserOrgAudit.java | 394 +++++++++++++
.../org/apache/usergrid/tools/OrgFixTool.java | 133 +++++
.../usergrid/tools/OrgUniqueIndexScanner.java | 476 +++++++++++++++
.../tools/OrganizationIndexMissingFix.java | 319 +++++++++++
7 files changed, 1998 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
new file mode 100644
index 0000000..d99d28f
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
@@ -0,0 +1,572 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.thrift.TBaseHelper;
+
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.PagingResultsIterator;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
+import org.apache.usergrid.persistence.cassandra.Serializers;
+import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.collect.BiMap;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.DynamicComposite;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.dce;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.ue;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+
+
+/**
+ * This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users then it
+ * deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row. If
+ * there exists more than one existing column then the one with the most recent timestamp wins and the other is
+ * deleted.
+ *
+ * If you want the run the tool on their cluster the following is what you need to do nohup java
+ * -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+ *
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value
+ * <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class ApplicationAuditOrgFix extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( ApplicationAuditOrgFix.class );
+
+ private static final String APP_VALUE = "app";
+
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+ Option appOption =
+ OptionBuilder.withArgName( "app" ).hasArg().isRequired( false ).withDescription( "Application uuid" )
+ .create( "app" );
+
+ options.addOption( appOption );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique checker" );
+
+
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Keyspace keyspaceforOrgAppFix = cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID );
+ Mutator<ByteBuffer> repairMutator = createMutator( cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID ), be );
+ int index = 0;
+ HashSet<UUID> orgUUIDs = getOrgUUIDS();
+
+
+ for ( UUID orgUUID : orgUUIDs ) {
+ OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( orgUUID );
+
+ if ( organizationInfo == null ) {
+
+ List<HColumn<DynamicComposite, ByteBuffer>> cols =
+ cass.getAllColumns( keyspaceforOrgAppFix, ENTITY_COMPOSITE_DICTIONARIES,
+ key( orgUUID, "connected_entities", "owns" ), dce, be );
+ if ( cols.size() == 0 ) {
+ logger.error( "There are no applications associated with this org: {}", orgUUID );
+ }
+ else {
+ Boolean orgFixed = false;
+ for ( HColumn<DynamicComposite, ByteBuffer> col : cols ) {
+ DynamicComposite dynamicComposite = col.getName();
+ UUID applicationId = ( UUID ) dynamicComposite.get( 0 );
+ logger.info( "The following applications are associated with this org uuid: {}. Trying to repair based on working application.",orgUUID ,applicationId );
+ ApplicationInfo applicationInfo = managementService.getApplicationInfo( applicationId );
+ if(applicationInfo == null){
+ logger.info("The following application uuid: {} returns null for org uuid: {}",applicationId,orgUUID);
+ }
+ else{
+ logger.info("The following application with information: {}, will be used to repair org uuid: {}.",applicationInfo,orgUUID);
+ }
+
+ }
+ }
+ }
+ else {
+
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "groups";
+ String uniqueValueKey = "path";
+ String uniqueValue = organizationInfo.getName();
+
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key(applicationId, collectionName, uniqueValueKey,uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
+ false );
+
+ if(cols.size() == 0){
+ logger.info("Organization uuid: {} with name: {} doesn't exist in ENTITY_UNIQUE. Repairing",organizationInfo.getUuid(),organizationInfo.getName());
+ addInsertToMutator( repairMutator, ENTITY_UNIQUE, key, orgUUID, null, createTimestamp() );
+ repairMutator.execute();
+ OrganizationInfo verificationOrganizationInfo = managementService.getOrganizationByName( organizationInfo.getName() );
+
+ if(verificationOrganizationInfo==null){
+ logger.error("Repair failed for rowkey: {} and org uuid: {}",key,orgUUID);
+ }
+ else if(verificationOrganizationInfo.getUuid().equals( organizationInfo.getUuid())&& verificationOrganizationInfo.equals( organizationInfo.getName() )){
+ logger.info( "Org name: {} with uuid is fixed: {}",organizationInfo.getUuid(),organizationInfo.getUuid() );
+ }
+ }
+ else {
+ Boolean correctUUIDFlag = false;
+ UUID alternateUUID = null;
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID uuid = Serializers.ue.fromByteBuffer( col.getName() );
+ if(uuid.equals( orgUUID )){
+ correctUUIDFlag = true;
+ }
+ alternateUUID = uuid;
+ }
+ if(!correctUUIDFlag){
+ logger.error("Found the same name: {} but different uuid: {} from the entity index: {}",new Object[]{organizationInfo.getName(),alternateUUID,organizationInfo.getUuid()});
+ }
+ else {
+ logger.info( "orgUUID: {} works", orgUUID );
+ }
+ }
+ }
+ }
+ logger.info( "Completed logging successfully. Found this many organizations: {}", orgUUIDs.size() );
+
+
+ // RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ // HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily(
+ // ENTITY_COMPOSITE_DICTIONARIES.getColumnFamily() )
+ // //not sure if I trust the lower two settings as it might iterfere with
+ // paging or set
+ // // arbitrary limits and what I want to retrieve.
+ // //That needs to be verified.
+ // .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+ //
+ //
+ // RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery,
+ // null, null );
+ //
+ // while ( rangeSlicesIterator.hasNext() ) {
+ // Row rangeSliceValue = rangeSlicesIterator.next();
+ //
+ //
+ // ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ // //Cassandra client library returns ByteBuffers that are views on top of a larger
+ // byte[]. These
+ // // ones return garbage data.
+ // //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ // String returnedRowKey =
+ // new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining
+ // (), Charset
+ // .defaultCharset() ).trim();
+ // logger.info( "HEre is a valid rowkey: {} ", returnedRowKey );
+ //
+ // //defensive programming, don't have to have to parse the string if it doesn't
+ // contain users.
+ //// if ( returnedRowKey.contains( "connected_entities" ) && returnedRowKey.contains(
+ // "owns" ) ) {
+ //// logger.info( "HEre is a valid rowkey: {} ", returnedRowKey );
+ //// }
+ // }
+
+
+ }
+
+ // //maybe put a byte buffer infront.
+ // RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ // HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily(
+ // ENTITY_COMPOSITE_DICTIONARIES.getColumnFamily() )
+ // //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // // arbitrary limits and what I want to retrieve.
+ // //That needs to be verified.
+ // .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+ //
+ //
+ // RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+ //
+ // while ( rangeSlicesIterator.hasNext() ) {
+ // Row rangeSliceValue = rangeSlicesIterator.next();
+ //
+ //
+ // ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ // //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These
+ // larger
+ // // ones return garbage data.
+ // //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ // String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf
+ // .remaining(),
+ // Charset.defaultCharset() ).trim();
+ //
+ // //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ // if (returnedRowKey.contains("connected_entities") && returnedRowKey.contains( "owns" )) {
+ //
+ // String[] parsedRowKey = returnedRowKey.split( ":" );
+ //
+ // if ( parsedRowKey.length < 3 ) {
+ // logger.error( "{} is a invalid row key, and unparseable due to length. Skipped...",
+ // returnedRowKey );
+ // continue;
+ // }
+ //
+ // parsedRowKey = garbageRowKeyParser( parsedRowKey );
+ //
+ // if ( parsedRowKey == null ) {
+ // logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ // continue;
+ // }
+ //
+ //
+ // orgUUIDs.add( UUID.fromString(parsedRowKey[0]) );
+ //
+ // ColumnSlice<ByteBuffer, ByteBuffer> columnSlice=rangeSliceValue.getColumnSlice();
+ // if ( columnSlice.getColumns().size() != 0 ) {
+ // List<HColumn<ByteBuffer, ByteBuffer>> cols=columnSlice.getColumns();
+ // for(HColumn<ByteBuffer,ByteBuffer> col: cols) {
+ // logger.info( "This is row key: {} 's column name: {} ",returnedRowKey,col );
+ //
+ // OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( UUID
+ // .fromString(parsedRowKey[0]) );
+ //
+ // logger.error("Probably blows up: {}.",ue.fromByteBuffer(col.getName()));
+ //// if(organizationInfo == null){
+ //// logger.error("Cannot retreive organization. check to see if can be repaired
+ // by application.");
+ //// }
+ // }
+ //
+ // }
+ //
+ //
+ //
+ //
+ //
+
+ //
+ // //if the rowkey contains more than 4 parts then it may have some garbage appended to the
+ // front.
+ // if ( parsedRowKey.length > 4 ) {
+ // parsedRowKey = garbageRowKeyParser( parsedRowKey );
+ //
+ // if ( parsedRowKey == null ) {
+ // logger.error( "{} is a invalid row key, and unparseable. Skipped...",
+ // returnedRowKey );
+ // continue;
+ // }
+ // }
+ // //if the rowkey contains less than four parts then it is completely invalid
+ // else if ( parsedRowKey.length < 4 ) {
+ // logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ // continue;
+ // }
+ //
+ // UUID applicationId = null;
+ // try {
+ // applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ // }
+ // catch ( Exception e ) {
+ // logger.error( "could not parse {} despite earlier parsing. Skipping...",
+ // parsedRowKey[0] );
+ // continue;
+ // }
+ // String collectionName = parsedRowKey[1];
+ // String uniqueValueKey = parsedRowKey[2];
+ // String uniqueValue = parsedRowKey[3];
+
+
+ // if ( collectionName.equals( "users" ) ) {
+
+ //ColumnSlice<ByteBuffer, ByteBuffer> columnSlice=rangeSliceValue.getColumnSlice();
+ //if ( columnSlice.getColumns().size() != 0 ) {
+ // List<HColumn<ByteBuffer, ByteBuffer>> cols=columnSlice.getColumns();
+
+ //entityStateLogger( applicationId,uniqueValue);
+ //}
+ // }
+ // }
+ // }
+
+
+ private HashSet<UUID> getOrgUUIDS() throws Exception {
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+ RelationManagerImpl relationManager =
+ ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+ Query query = new Query();
+ query.setLimit( 1000 );
+ query.setResultsLevel( Results.Level.IDS );
+
+ HashSet<UUID> orgUUIDs = new HashSet();
+
+ Results results = relationManager.searchCollection( "groups", query );
+
+ PagingResultsIterator pagingResultsIterator = new PagingResultsIterator( results );
+ while ( pagingResultsIterator.hasNext() ) {
+ orgUUIDs.add( ( UUID ) pagingResultsIterator.next() );
+ }
+ logger.info( "Found {} number of orgs", orgUUIDs.size() );
+ return orgUUIDs;
+ }
+
+
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 3 ) {
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
+
+ private void entityStateLogger( final UUID applicationUUID, final String uniqueValue ) throws Exception {
+
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+
+ ApplicationInfo applicationInfo = managementService.getApplicationInfo( applicationUUID );
+
+ if ( applicationInfo == null ) {
+ if ( !applicationUUID.equals( UUID.fromString( "00000000-0000-0000-0000-000000000010" ) ) ) {
+
+ logger.error( "Application uuid: {} and name: {} has return null.", applicationUUID, uniqueValue );
+ }
+ }
+ else {
+ OrganizationInfo organizationInfo = managementService.getOrganizationForApplication( applicationUUID );
+ if ( organizationInfo == null ) {
+ if ( !applicationUUID.equals( UUID.fromString( "00000000-0000-0000-0000-000000000001" ) ) ) {
+
+ logger.error (
+ "Application uuid: {} with name: {} is lost and has no organizations associated with it.",
+ applicationUUID, applicationInfo.getName() );
+ }
+ }
+ else {
+ EntityRef orgRef = new SimpleEntityRef( "group", organizationInfo.getUuid() );
+ orgVerification (
+ applicationUUID, uniqueValue, em, orgRef, organizationInfo.getUuid(),
+ "Application with uuid: {} with name: {} is missing their organization: {}" );
+ }
+ }
+ }
+
+
+ private void orgVerification( final UUID applicationUUID, final String uniqueValue, final EntityManager em,
+ final EntityRef orgRef, final UUID uuid, final String s2 ) throws Exception {
+ try {
+ em.getDictionaryAsMap( orgRef, "orgProperties" );
+
+ Object[] loggingObject = new Object[3];
+ loggingObject[0] = applicationUUID;
+ loggingObject[1] = uniqueValue;
+ loggingObject[2] = uuid;
+ logger.info( "Application with uuid: {} and name: {} was returned from orgUUID: {} ", loggingObject );
+ }
+ catch ( EntityNotFoundException enfe ) {
+ Object[] notFoundLogger = new Object[3];
+ notFoundLogger[0] = applicationUUID;
+ notFoundLogger[1] = uniqueValue;
+ notFoundLogger[2] = uuid;
+
+ logger.error( s2, uniqueValue, uuid );
+
+ BiMap<UUID, String> applicationBiMap = managementService.getApplicationsForOrganization( uuid );
+ if ( applicationBiMap == null || applicationBiMap.size() == 0 ) {
+ logger.error( "Applications didn't return for orgUUID: {}", uuid );
+ }
+ else if ( applicationBiMap.size() > 0 ) {
+ for ( Map.Entry<UUID, String> app : applicationBiMap.entrySet() ) {
+ Object[] loggingObject = new Object[3];
+ loggingObject[0] = app.getKey();
+ loggingObject[1] = app.getValue();
+ loggingObject[2] = uuid;
+ logger.info ( "Application with uuid: {} and name: {} was returned from orgUUID: {} ",
+ loggingObject );
+ }
+ }
+ }
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
+ UUID applicationId = UUID.fromString( line.getOptionValue( APP_VALUE ) );
+ String collectionName = "applications"; //line.getOptionValue( COLLECTION_ARG );
+ String uniqueValueKey = "name"; //line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+
+ // List<HColumn<ByteBuffer, ByteBuffer>> cols = cass.getColumns( cass.getApplicationKeyspace(
+ // applicationId), ENTITY_UNIQUE, key, null, null, 1000,
+ // false );
+
+
+ // if ( cols.size() == 0 ) {
+ // logger.error( "This row key: {} has zero columns", key.toString() );
+ // }
+
+ entityStateLogger( applicationId, uniqueValue );
+ }
+
+
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ logger.error( "{} is unparsable", garbageString );
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
index eeb2575..d6ac6c5 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserAudit.java
@@ -20,6 +20,8 @@ package org.apache.usergrid.tools;
import java.io.File;
import java.nio.ByteBuffer;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.UUID;
import org.codehaus.jackson.JsonNode;
@@ -32,8 +34,13 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+
+import com.google.common.collect.BiMap;
import me.prettyprint.hector.api.beans.HColumn;
@@ -162,6 +169,8 @@ public class ManagementUserAudit extends ToolBase {
Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ EntityManagerImpl emi = ( EntityManagerImpl ) em;
+
List<HColumn<ByteBuffer, ByteBuffer>> cols =
cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, be, be );
@@ -184,26 +193,105 @@ public class ManagementUserAudit extends ToolBase {
logger.error( "Email: {} doesn't exist in ENTITY_UNIQUE.", extractedEmail );
}
else {
- logger.error( "Email: {} has {} number of duplicate columns in ENTITY_UNIQUE", extractedEmail,
- cols.size() );
+ logger.warn( "Email: {} has {} number of duplicate columns in ENTITY_UNIQUE.Attempting Repair.",
+ extractedEmail, cols.size() );
UUID uuid = null;
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- uuid = ue.fromByteBuffer( col.getName() );
- Entity entity = em.get( uuid );
- if ( entity == null ) {
- logger.error( "Email: {} with duplicate uuid: {} doesn't exist in ENTITY_PROPERTIES.",
- extractedEmail, uuid );
+
+ //Only handles the case if there are two duplicates, if there are more then expand the tool. None
+ // have been found so far.
+
+ //if(cols.size()==2){
+ uuid = ue.fromByteBuffer( cols.get( 0 ).getName() );
+ Entity entity = em.get( uuid );
+ Long created = entity.getCreated();
+ UserInfo adminUserInfo = managementService.getAdminUserByUuid( uuid );
+
+
+ for ( int index = 1; index < cols.size(); index++ ) {
+ UUID duplicateEmailUUID = ue.fromByteBuffer( cols.get( index ).getName() );
+ Entity duplicateEntity = em.get( duplicateEmailUUID );
+
+ //if the dup is newer than the first entry
+ if ( created < duplicateEntity.getCreated() ) {
+ BiMap<UUID, String> uuidStringOrgBiMap =
+ managementService.getOrganizationsForAdminUser( duplicateEmailUUID );
+ BiMap<String, UUID> stringUUIDBiMap = uuidStringOrgBiMap.inverse();
+ for ( String orgName : stringUUIDBiMap.keySet() ) {
+ logger.warn( "Adding admin user: {} to organization: {}", adminUserInfo, orgName );
+ OrganizationInfo organizationInfo = managementService
+ .getOrganizationByUuid( stringUUIDBiMap.get( orgName ) );
+ managementService.addAdminUserToOrganization
+ (adminUserInfo,organizationInfo , false );
+ }
+ logger.warn( "Deleting duplicated uuid: {}.", duplicateEmailUUID );
+ emi.deleteEntity( duplicateEmailUUID );
}
- else {
- Object[] loggerObject = new Object[3];
- loggerObject[0] = extractedEmail;
- loggerObject[1] = uuid;
- loggerObject[2] = entity;
- logger.info( "Email: {} with duplicate uuid: {} with the following data: {} exists in "
- + "ENTITY_PROPERTIES", extractedEmail, uuid );
+ else if ( created > duplicateEntity.getCreated() ) {
+ logger.info("older column was returned later from cassandra");
+ BiMap<UUID, String> uuidStringOrgBiMap = managementService.getOrganizationsForAdminUser( uuid );
+ BiMap<String, UUID> stringUUIDBiMap = uuidStringOrgBiMap.inverse();
+ adminUserInfo = managementService.getAdminUserByUuid( duplicateEmailUUID );
+
+
+ for ( String orgName : stringUUIDBiMap.keySet() ) {
+ logger.warn( "Adding admin user: {} to organization: {}", adminUserInfo, orgName );
+ OrganizationInfo organizationInfo = managementService.getOrganizationByUuid(
+ stringUUIDBiMap.get( orgName ) );
+ managementService.addAdminUserToOrganization(adminUserInfo,organizationInfo , false );
+ }
+ logger.warn( "Deleting duplicated uuid: {}.", uuid );
+ emi.deleteEntity( uuid );
+ created = duplicateEntity.getCreated();
+ uuid = duplicateEmailUUID;
}
}
+
+ // }
+
+ // for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ // uuid = ue.fromByteBuffer( col.getName() );
+ // //managementService.get
+ // entity = em.get( uuid );
+ // adminUserInfo = managementService.getAdminUserByUuid( uuid );
+ //
+ // Map<String, Object>
+ // userOrganizationData = managementService.getAdminUserOrganizationData(
+ // adminUserInfo, true );
+ //
+ //
+ // if ( entity == null ) {
+ // logger.error( "Email: {} with duplicate uuid: {} doesn't exist in
+ // ENTITY_PROPERTIES.",
+ // extractedEmail, uuid );
+ // }
+ // else {
+ //
+ //
+ //
+ //
+ // Object[] loggerObject = new Object[4];
+ // loggerObject[0] = extractedEmail;
+ // loggerObject[1] = uuid;
+ // loggerObject[2] = adminUserInfo;
+ // loggerObject[3] = userOrganizationData;
+ // logger.warn( "Email: {} with duplicate uuid: {} with the following data:
+ // {} and organizational data: {} exists in "
+ // + "ENTITY_PROPERTIES", loggerObject);
+ //
+
+ // Set<String> dictionaries = emi.getDictionaryNames( entity );
+ // if ( dictionaries != null ) {
+ // for ( String dictionary : dictionaries ) {
+ // Set<Object> values = emi.getDictionaryAsSet( entity, dictionary );
+ // logger.warn("The uuid: {} has the following dictionary: {} with the following values: {}.",new Object[] {uuid,dictionary,values});
+ // for ( Object value : values ) {
+ // emi.batchUpdateDictionary( m, entity, dictionary, value, true, timestampUuid );
+ // }
+ // }
+ // }
+ // }
}
}
}
}
+//}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
index 5d075fc..09e3202 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserIndexMissingFix.java
@@ -286,7 +286,7 @@ public class ManagementUserIndexMissingFix extends ToolBase {
//preconditions should be satisfied before trying this method. Preconditions satisfy that
//uuid doesn't have valid data anywhere else but in INDEX for specific email. May still have the
//broken uuid else where but not of concern to users.
-
+
Object index_key = key( index_name, bucketId );
logger.info( "Deleting the following rowkey: {} from ENTITY_INDEX.", index_key );
addDeleteToMutator( m, ENTITY_INDEX, index_key, columnName, createTimestamp() );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserOrgAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserOrgAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserOrgAudit.java
new file mode 100644
index 0000000..8ef94fc
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ManagementUserOrgAudit.java
@@ -0,0 +1,394 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.thrift.TBaseHelper;
+
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.collect.BiMap;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+
+
+/**
+ * This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users then it
+ * deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row. If
+ * there exists more than one existing column then the one with the most recent timestamp wins and the other is
+ * deleted.
+ *
+ * If you want the run the tool on their cluster the following is what you need to do nohup java
+ * -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+ *
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value
+ * <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class ManagementUserOrgAudit extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( ManagementUserOrgAudit.class );
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique checker" );
+
+
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ if ( line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ deleteInvalidValuesForUniqueProperty( m, line );
+ }
+ else {
+ //maybe put a byte buffer infront.
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
+ //That needs to be verified.
+ .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+
+
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
+
+
+ ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
+ // ones return garbage data.
+ //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
+ Charset.defaultCharset() ).trim();
+
+
+ //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ if (returnedRowKey.contains("email") && returnedRowKey.contains( "users" ) && returnedRowKey.contains( MANAGEMENT_APPLICATION_ID.toString() )) {
+
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+
+ //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
+ if ( parsedRowKey.length > 4 ) {
+ parsedRowKey = garbageRowKeyParser( parsedRowKey );
+
+ if ( parsedRowKey == null ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ continue;
+ }
+ }
+ //if the rowkey contains less than four parts then it is completely invalid
+ else if ( parsedRowKey.length < 4 ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ continue;
+ }
+
+ UUID applicationId = null;
+ try {
+ applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ }
+ catch ( Exception e ) {
+ logger.error( "could not parse {} despite earlier parsing. Skipping...", parsedRowKey[0] );
+ continue;
+ }
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
+
+
+ if ( collectionName.equals( "users" ) ) {
+
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice=rangeSliceValue.getColumnSlice();
+ //if ( columnSlice.getColumns().size() != 0 ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> cols=columnSlice.getColumns();
+
+ entityStateLogger( uniqueValue, cols );
+ }
+ }
+ }
+ }
+ logger.info( "Completed logging successfully" );
+ }
+
+
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 4 ) {
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
+
+ private void entityStateLogger( final String uniqueValue, final List<HColumn<ByteBuffer, ByteBuffer>> cols ) throws Exception {
+
+ UserInfo userInfo = null;
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+
+ try {
+ userInfo = managementService.getAdminUserByEmail( uniqueValue );
+ }catch(Exception e){
+ logger.error("threw exception when looking up email: {}",uniqueValue);
+ e.printStackTrace();
+ }
+ if(userInfo==null) {
+ if(cols!=null){
+ if(cols.size()>1){
+ for(HColumn<ByteBuffer, ByteBuffer> col : cols) {
+ logger.warn( "This uuid: {} is associated with this duplicated email {}", ue.fromByteBuffer( col.getName()),uniqueValue );
+ }
+
+ }
+ if(cols.size()==1){
+ logger.error( "Management user with uuid: {} and email: {} is broken.",ue.fromByteBuffer( cols.get( 0 ).getName()), uniqueValue );
+ }
+ else{
+ logger.error( "Management user with email: {} is broken and has no uuid's associated with it",uniqueValue );
+ }
+ }
+ }
+ else {
+ Results orgResults =
+ em.getCollection( new SimpleEntityRef( User.ENTITY_TYPE, userInfo.getUuid() ), "groups", null, 10000, Results.Level.REFS,
+ false );
+
+ if(orgResults.getRefs().size() == 1){
+ EntityRef orgRef = orgResults.getRef();
+ orgVerification( uniqueValue, em, orgRef, orgRef.getUuid(),
+ "Management user with email: {} is present but cannot login due to missing their only organization: {}" );
+ }
+ else {
+ for ( EntityRef orgRef : orgResults.getRefs() ) {
+ orgVerification( uniqueValue, em, orgRef, orgRef.getUuid(),
+ "Management user with email: {} is present with multiple orgs but is missing the following orgUUID: {}" );
+
+ }
+ }
+ }
+
+ }
+
+
+ private void orgVerification( final String uniqueValue, final EntityManager em, final EntityRef orgRef,
+ final UUID uuid, final String s2 ) throws Exception {
+ try {
+ em.getDictionaryAsMap( orgRef, "orgProperties" );
+ OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( uuid );
+
+ if(organizationInfo == null) {
+ logger.error( "The following email works: {} but the orgUUID: {} returns null.",uniqueValue,uuid );
+ }
+ else {
+
+ Object[] loggingObject = new Object[3];
+ loggingObject[0] = uniqueValue;
+ loggingObject[1] = organizationInfo.getName();
+ loggingObject[2] = organizationInfo.getUuid();
+
+ logger.info( "The following email works: {} with the following orgname: {} and orgUUID: {}",
+ loggingObject );
+ }
+ }
+ catch ( EntityNotFoundException enfe ) {
+ logger.error( s2, uniqueValue, uuid );
+
+ BiMap<UUID,String> applicationBiMap = managementService.getApplicationsForOrganization( uuid );
+ if(applicationBiMap==null || applicationBiMap.size() == 0){
+ logger.error("Applications didn't return for orgUUID: {}",uuid);
+ }
+ else if (applicationBiMap.size()>0){
+ for ( Map.Entry<UUID, String> app : applicationBiMap.entrySet() ) {
+ Object[] loggingObject = new Object[3];
+ loggingObject[0] = app.getKey();
+ loggingObject[1] = app.getValue();
+ loggingObject[2] = uuid;
+ logger.info( "Application with uuid: {} and name: {} was returned from orgUUID: {} ",loggingObject );
+
+ }
+ }
+
+ }
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "users"; //line.getOptionValue( COLLECTION_ARG );
+ String uniqueValueKey = "email"; //line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key( applicationId, collectionName,"email", uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = cass.getColumns( cass.getApplicationKeyspace( applicationId), ENTITY_UNIQUE, key, null, null, 1000,
+ false );
+
+
+ if ( cols.size() == 0 ) {
+ logger.error( "This row key: {} has zero columns", key.toString() );
+ }
+
+ entityStateLogger( uniqueValue, cols );
+ }
+
+
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ logger.error( "{} is unparsable", garbageString );
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/OrgFixTool.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/OrgFixTool.java b/stack/tools/src/main/java/org/apache/usergrid/tools/OrgFixTool.java
new file mode 100644
index 0000000..e27aedc
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/OrgFixTool.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.util.List;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
+import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
+import org.apache.usergrid.persistence.query.ir.result.SecondaryIndexSliceParser;
+import org.apache.usergrid.persistence.query.ir.result.UUIDColumn;
+
+import me.prettyprint.hector.api.beans.DynamicComposite;
+
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.dce;
+
+
+/**
+ * Created by ApigeeCorporation on 12/15/15.
+ */
+public class OrgFixTool extends ToolBase {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( OrgFixTool.class );
+
+ private static final String FILE_PATH = "file";
+
+ private static final String DUPLICATE_EMAIL = "dup";
+
+ private static final String ROW_KEY = "row";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+
+ Option duplicate_email = OptionBuilder.withArgName( DUPLICATE_EMAIL ).hasArg().isRequired( false )
+ .withDescription( "duplicate org uuid to examine" )
+ .create( DUPLICATE_EMAIL );
+ options.addOption( duplicate_email );
+
+ return options;
+ }
+
+
+ @Override
+ public void runTool( final CommandLine line ) throws Exception {
+ startSpring();
+
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID);
+ RelationManagerImpl relationManager =
+ ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+
+ Query query = new Query();
+ query.setEntityType( "group" );
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Results.Level.REFS );
+
+ if(line.getOptionValue( "dup" )!=null) {
+ query.addEqualityFilter( "uuid",line.getOptionValue( "dup" ) );
+
+ logger.info( "This is what returns from the call to managements: {}",
+ managementService.getApplicationsForOrganization( UUID.fromString( line.getOptionValue( "dup" ) ) ) );
+ //managementService.getOrganizationByIdentifier( )
+
+ }
+ else {
+ List<ScanColumn> scanColumnList = relationManager.searchRawCollection( "groups", query ); //em.searchCollection( em.getApplicationRef(), "groups", query );
+
+ for ( ScanColumn scanColumn : scanColumnList ) {
+ logger.info( "This is what you get back: {}", scanColumn );
+ // SecondaryIndexSliceParser.SecondaryIndexColumn secondaryIndexColumn =
+ // ( SecondaryIndexSliceParser.SecondaryIndexColumn ) scanColumn;
+ UUIDColumn uuidColumn = ( UUIDColumn ) scanColumn;
+
+ //byte buffer from the scan column is the column name needed to delete from ENTITY_INDEX
+ //DynamicComposite columnName = dce.fromByteBuffer( uuidColumn.getByteBuffer() );
+
+
+ //logger.info("This is what you get back from the buffer: {}, from a buffer of size: {}",sc);
+
+ logger.info( "This is what returns from the call to managements: {}",
+ managementService.getApplicationsForOrganization( uuidColumn.getUUID() ) );
+
+
+ // if(em.get((UUID ) columnName.get( 2 ) ) == null){
+ // logger.error( "This organization is broken and doesn't appear in entity Properties: {}",columnName );
+ // }
+ // else{
+ // logger.info("This organization works: {}",columnName);
+ // }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2a9fb89/stack/tools/src/main/java/org/apache/usergrid/tools/OrgUniqueIndexScanner.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/OrgUniqueIndexScanner.java b/stack/tools/src/main/java/org/apache/usergrid/tools/OrgUniqueIndexScanner.java
new file mode 100644
index 0000000..8c6fb33
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/OrgUniqueIndexScanner.java
@@ -0,0 +1,476 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.thrift.TimedOutException;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.thrift.TBaseHelper;
+
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.PagingResultsIterator;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
+import org.apache.usergrid.persistence.entities.Group;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
+import org.apache.usergrid.persistence.query.ir.result.UUIDColumn;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+
+
+/**
+ * This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users then it
+ * deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row. If
+ * there exists more than one existing column then the one with the most recent timestamp wins and the other is
+ * deleted.
+ *
+ * If you want the run the tool on their cluster the following is what you need to do nohup java
+ * -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+ *
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value
+ * <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class OrgUniqueIndexScanner extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( OrgUniqueIndexScanner.class );
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique index cleanup" );
+
+
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+// EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+//
+// RelationManagerImpl relationManager =
+// ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+//
+// Query query = new Query();
+// query.setEntityType( "group" );
+// query.setLimit( MAX_ENTITY_FETCH );
+// query.setResultsLevel( Results.Level.REFS );
+//
+// Results r = relationManager.searchCollection( "groups",query );
+
+ HashSet<UUID> orgUUIDS = getOrgUUIDS();
+ int index = 0;
+
+
+ if ( line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ deleteInvalidValuesForUniqueProperty( m, line,orgUUIDS );
+ }
+ else {
+
+
+ //maybe put a byte buffer infront.
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
+ //That needs to be verified.
+ .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+
+
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
+
+
+ ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
+ // ones return garbage data.
+ //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
+ Charset.defaultCharset() ).trim();
+
+
+ //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ if ( returnedRowKey.contains( "groups" ) && returnedRowKey
+ .contains( MANAGEMENT_APPLICATION_ID.toString() ) ) {
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+
+ //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
+ if ( parsedRowKey.length > 4 ) {
+ parsedRowKey = garbageRowKeyParser( parsedRowKey );
+
+ if ( parsedRowKey == null ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ continue;
+ }
+ }
+ //if the rowkey contains less than four parts then it is completely invalid
+ else if ( parsedRowKey.length < 4 ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
+ continue;
+ }
+
+ UUID applicationId = null;
+ try {
+ applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ }
+ catch ( Exception e ) {
+ logger.error( "could not parse {} despite earlier parsing. Skipping...", parsedRowKey[0] );
+ continue;
+ }
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
+
+
+ //if ( collectionName.equals( "users" ) ) {
+
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ //if ( columnSlice.getColumns().size() != 0 ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+
+
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,
+ returnedRowKey,orgUUIDS);
+ index++;
+ // }
+ // }
+ //}
+ }
+ }
+ }
+
+
+ logger.info("Completed repair successfully.Scanned over {} number of orgs",index);
+}
+
+
+
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 4 ) {
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
+
+ private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue ) throws Exception {
+ logger.debug( "Found 0 uuid's associated with {} Deleting row.", uniqueValue );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, timestamp );
+ mutator.execute();
+ return;
+ }
+
+
+ private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue,
+ final List<HColumn<ByteBuffer, ByteBuffer>> cols, String rowKey,HashSet<UUID> orgUUIDS) throws Exception {
+ Boolean cleanup = false;
+ EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+
+ int numberOfColumnsDeleted = 0;
+ //these columns all come from the same row key, which means they each belong to the same row key identifier
+ //thus mixing and matching them in the below if cases won't matter.
+ Entity[] entities = new Entity[cols.size()];
+ int numberOfRetrys = 8;
+ int numberOfTimesRetrying = 0;
+
+ int index = 0;
+
+
+ if ( cols.size() == 1 ) {
+ UUID uuid = null;
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ uuid = ue.fromByteBuffer( col.getName() );
+ }
+
+ if ( em.get( uuid ) == null ) {
+ logger.error( "OrgName: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES.", uniqueValue, uuid );
+
+ if(orgUUIDS.contains( uuid )){
+ logger.error( "OrgName: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES but exists in ENTITY_INDEX.Repairing...", uniqueValue, uuid );
+
+ Group organizationEntity = new Group();
+ organizationEntity.setPath( uniqueValue );
+ try {
+ em.create( uuid, Group.ENTITY_TYPE, organizationEntity.getProperties() );
+ organizationEntity = em.get( uuid, Group.class );
+ if ( organizationEntity == null ) {
+ logger.error( "OrgName: {} with uuid: {} could not be created", uniqueValue, uuid );
+ }
+ else{
+// logger.info( "OrgName: {} with uuid: {} was fixed.", uniqueValue,
+// uuid );
+ }
+ }catch(Exception e){
+ logger.error("Could not create: {} because of {}",uniqueValue,e.getMessage());
+ }
+
+ }
+ else{
+ logger.error( "OrgName: {} with uuid: {} doesn't exist in ENTITY_PROPERTIES OR ENTITY INDEX. Leaving alone.", uniqueValue, uuid );
+
+ }
+ }
+ else {
+// logger.info( "OrgName: {} with uuid: {} exists in ENTITY_PROPERTIES for ENTITY_UNIQUE.", uniqueValue,
+// uuid );
+ }
+ }
+
+ }
+
+
+ private HashSet<UUID> getOrgUUIDS() throws Exception {
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+ RelationManagerImpl relationManager =
+ ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+ Query query = new Query();
+ query.setLimit( 1000 );
+ query.setResultsLevel( Results.Level.IDS );
+
+ HashSet<UUID> orgUUIDs = new HashSet( );
+
+ Results results = relationManager.searchCollection( "groups",query );
+
+ PagingResultsIterator pagingResultsIterator = new PagingResultsIterator( results );
+ while(pagingResultsIterator.hasNext()){
+ orgUUIDs.add((UUID) pagingResultsIterator.next());
+ }
+ logger.info( "Found {} number of orgs",orgUUIDs.size() );
+ return orgUUIDs;
+ }
+
+
+ private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
+ Entity entity = unverifiedEntity;
+ if ( entity != null && entity.getModified() == null ) {
+ if ( entity.getCreated() != null ) {
+ logger.debug( "{} has no modified. Subsituting created timestamp for their modified timestamp.Manually "
+ + "adding one for comparison purposes", entity.getUuid() );
+ entity.setModified( entity.getCreated() );
+ return entity;
+ }
+ else {
+ logger.error( "Found no created or modified timestamp. Please remake the following entity: {}."
+ + " Setting both created and modified to 1", entity.getUuid().toString() );
+ entity.setCreated( 1L );
+ entity.setModified( 1L );
+ return entity;
+ }
+ }
+ return entity;
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line,
+ final HashSet<UUID> orgUUIDS ) throws Exception {
+ UUID applicationId = MANAGEMENT_APPLICATION_ID;
+ String collectionName = "groups";
+ String uniqueValueKey = "path";
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE);
+
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key(applicationId, collectionName, uniqueValueKey,uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
+ false );
+
+
+ if ( cols.size() == 0 ) {
+ logger.error( "This row key: {} has zero columns. Leaving alone.", key.toString() );
+ }
+ else if(cols.size() > 1){
+ logger.error( "This row key: {} has duplicate entries columns. Leaving alone.", key.toString() );
+
+ }
+
+// EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
+//
+// RelationManagerImpl relationManager =
+// ( RelationManagerImpl ) em.getRelationManager( ref( MANAGEMENT_APPLICATION_ID ) );
+
+// Query query = new Query();
+// query.setEntityType( "group" );
+// query.setLimit( MAX_ENTITY_FETCH );
+// query.setResultsLevel( Results.Level.REFS );
+//
+// Results r = relationManager.searchCollection( "groups",query );
+//
+// managementService.getOrganizationByUuid( )
+//
+// logger.info("Does this work?: {}",r);
+
+
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols, key.toString(),orgUUIDS);
+ }
+
+
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ logger.error( "{} is unparsable", garbageString );
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+}
[4/4] usergrid git commit: Added way for applications to be
reconnected with organizations and organizations to be reconnected to
applications.
Posted by gr...@apache.org.
Added way for applications to be reconnected with organizations and organizations to be reconnected to applications.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2627dbbf
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2627dbbf
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2627dbbf
Branch: refs/heads/unique_index_logging
Commit: 2627dbbfe7c74b26237f2b5f8b6dedb19347a6d2
Parents: 00c2d05
Author: George Reyes <gr...@apache.org>
Authored: Fri Dec 18 11:35:12 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Dec 18 11:35:12 2015 -0800
----------------------------------------------------------------------
.../apache/usergrid/tools/ApplicationAudit.java | 21 ++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2627dbbf/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java
index 037dc8f..cf702d4 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java
@@ -39,6 +39,7 @@ import org.apache.usergrid.management.cassandra.ManagementServiceImpl;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.security.AuthPrincipalType;
@@ -293,12 +294,6 @@ public class ApplicationAudit extends ToolBase {
logger.error("Org name: {} and uuid: {} with App uuid: {} and name: {} repair has failed.",
new Object[]{organizationInfo.getName(),organizationInfo.getUuid(),applicationUUID,uniqueValue});
}
-
-// ManagementServiceImpl managementServiceImpl = ( ManagementServiceImpl ) managementService;
-// managementServiceImpl.writeUserToken( MANAGEMENT_APPLICATION_ID, applicationEntity, encryptionService
-// .plainTextCredentials( generateOAuthSecretKey( AuthPrincipalType.APPLICATION ), null,
-// MANAGEMENT_APPLICATION_ID ) );
-// managementService.addApplicationToOrganization( organizationInfo.getUuid(), applicationUUID);
}
@@ -320,7 +315,21 @@ public class ApplicationAudit extends ToolBase {
logger.error("Org name: {} and uuid: {} is no longer associated with App uuid: {} and name: {}. Restablishing connection...",
new Object[]{organizationInfo.getName(),organizationInfo.getUuid(),applicationUUID,applicationInfo.getName()});
+ //add the application to organization, then add the organization to application.
managementService.addApplicationToOrganization( organizationInfo.getUuid(), applicationUUID);
+ //need to turn this into entity_refs.
+ em.createConnection(new SimpleEntityRef( "application_info", applicationUUID ) ,"owns",new SimpleEntityRef( "group",organizationInfo.getUuid() ) );
+
+ organizationInfo = managementService.getOrganizationForApplication( applicationUUID );
+
+ if(organizationInfo!=null){
+ logger.info("Org name: {} and uuid: {} with App uuid: {} and name: {} connection has been re-established.",
+ new Object[]{organizationInfo.getName(),organizationInfo.getUuid(),applicationUUID,applicationInfo.getName()});
+ }
+ else{
+ logger.error("Org name: {} and uuid: {} is no longer associated with App uuid: {} and name: {}. Could not be re-estabished",
+ new Object[]{organizationInfo.getName(),organizationInfo.getUuid(),applicationUUID,applicationInfo.getName()});
+ }
}
}
[3/4] usergrid git commit: Added org fix to ApplicationAuditOrgFix
and confirmed fix. also added application re-establishing and reconnecting
logic to tool for regular application audit.
Posted by gr...@apache.org.
Added org fix to ApplicationAuditOrgFix and confirmed fix. also added application re-establishing and reconnecting logic to tool for regular application audit.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/00c2d057
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/00c2d057
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/00c2d057
Branch: refs/heads/unique_index_logging
Commit: 00c2d0571db34dd99f7ed4fd7a7161a6ca19492f
Parents: d2a9fb8
Author: George Reyes <gr...@apache.org>
Authored: Fri Dec 18 11:16:21 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Dec 18 11:16:21 2015 -0800
----------------------------------------------------------------------
.../apache/usergrid/tools/ApplicationAudit.java | 65 ++++++++++++++++++--
.../usergrid/tools/ApplicationAuditOrgFix.java | 44 ++++++++++---
2 files changed, 94 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/00c2d057/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java
index 3a025cb..037dc8f 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAudit.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.tools;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -34,10 +35,13 @@ import org.apache.thrift.TBaseHelper;
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.cassandra.ManagementServiceImpl;
+import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
+import org.apache.usergrid.security.AuthPrincipalType;
import org.apache.usergrid.utils.UUIDUtils;
import com.google.common.collect.BiMap;
@@ -257,7 +261,47 @@ public class ApplicationAudit extends ToolBase {
if(applicationInfo==null){
if( !applicationUUID.equals( UUID.fromString( "00000000-0000-0000-0000-000000000010" ) )) {
- logger.error( "Application uuid: {} and name: {} has return null.", applicationUUID, uniqueValue );
+ //logger.error( "Application uuid: {} and name: {} has return null.", applicationUUID, uniqueValue );
+
+ String orgName = uniqueValue.split( "/" )[0];
+ OrganizationInfo organizationInfo = managementService.getOrganizationByName( orgName );
+ if(organizationInfo == null) {
+
+ logger.error( "Application uuid: {} with name: {} returned null and org name: {} cannot be found.",
+ new Object[]{applicationUUID, uniqueValue,orgName });
+ }
+ else{
+ logger.error("Org name: {} and uuid: {} is no longer associated with App uuid: {} and name: {} because APP NO LONGER EXISTS. RECREATING.",
+ new Object[]{organizationInfo.getName(),organizationInfo.getUuid(),applicationUUID,uniqueValue});
+
+
+
+
+ Map<String,Object> applicationProperties = new HashMap<String, Object>( );
+ applicationProperties.put( "name",uniqueValue );
+ try {
+ em.create( applicationUUID, ManagementServiceImpl.APPLICATION_INFO, applicationProperties);
+ }catch(Exception e){
+ logger.error("Found the following error: {} when trying to recreate.",e.getMessage());
+ }
+ applicationInfo = managementService.getApplicationInfo( applicationUUID );
+ if(applicationInfo!=null){
+ logger.info("Org name: {} and uuid: {} with App uuid: {} and name: {} has been Repaired.",
+ new Object[]{organizationInfo.getName(),organizationInfo.getUuid(),applicationInfo.getId(),applicationInfo.getName()});
+ }
+ else{
+ logger.error("Org name: {} and uuid: {} with App uuid: {} and name: {} repair has failed.",
+ new Object[]{organizationInfo.getName(),organizationInfo.getUuid(),applicationUUID,uniqueValue});
+ }
+
+// ManagementServiceImpl managementServiceImpl = ( ManagementServiceImpl ) managementService;
+// managementServiceImpl.writeUserToken( MANAGEMENT_APPLICATION_ID, applicationEntity, encryptionService
+// .plainTextCredentials( generateOAuthSecretKey( AuthPrincipalType.APPLICATION ), null,
+// MANAGEMENT_APPLICATION_ID ) );
+// managementService.addApplicationToOrganization( organizationInfo.getUuid(), applicationUUID);
+ }
+
+
}
}
else{
@@ -265,9 +309,20 @@ public class ApplicationAudit extends ToolBase {
if(organizationInfo==null) {
if(!applicationUUID.equals( UUID.fromString( "00000000-0000-0000-0000-000000000001"))) {
- logger.error(
- "Application uuid: {} with name: {} is lost and has no organizations associated with it.",
- applicationUUID, applicationInfo.getName() );
+ String orgName = applicationInfo.getName().split( "/" )[0];
+ organizationInfo = managementService.getOrganizationByName( orgName );
+ if(organizationInfo == null) {
+
+ logger.error( "Application uuid: {} with name: {} is lost and org name: {} cannot be found.",
+ new Object[]{applicationUUID, applicationInfo.getName(),orgName });
+ }
+ else{
+ logger.error("Org name: {} and uuid: {} is no longer associated with App uuid: {} and name: {}. Restablishing connection...",
+ new Object[]{organizationInfo.getName(),organizationInfo.getUuid(),applicationUUID,applicationInfo.getName()});
+
+ managementService.addApplicationToOrganization( organizationInfo.getUuid(), applicationUUID);
+
+ }
}
}
else{
@@ -309,7 +364,7 @@ public class ApplicationAudit extends ToolBase {
loggingObject[0] = app.getKey();
loggingObject[1] = app.getValue();
loggingObject[2] = uuid;
- logger.info( "Application with uuid: {} and name: {} was returned from orgUUID: {} ",loggingObject );
+ logger.info( "Application with uuid: {} and name: {} was returned from orgUUID: {} but that orgUUID cannot be found.",loggingObject );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/00c2d057/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
index d99d28f..0c7ad01 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ApplicationAuditOrgFix.java
@@ -43,6 +43,7 @@ import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.cassandra.RelationManagerImpl;
import org.apache.usergrid.persistence.cassandra.Serializers;
+import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.utils.UUIDUtils;
@@ -118,11 +119,11 @@ public class ApplicationAuditOrgFix extends ToolBase {
options.addOption( hostOption );
- Option appOption =
- OptionBuilder.withArgName( "app" ).hasArg().isRequired( false ).withDescription( "Application uuid" )
- .create( "app" );
-
- options.addOption( appOption );
+// Option appOption =
+// OptionBuilder.withArgName( "app" ).hasArg().isRequired( false ).withDescription( "Application uuid" )
+// .create( "app" );
+//
+// options.addOption( appOption );
Option entityUniquePropertyValue =
OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
@@ -148,11 +149,17 @@ public class ApplicationAuditOrgFix extends ToolBase {
// go through each collection and audit the values
- Keyspace ko = cass.getUsergridApplicationKeyspace();
Keyspace keyspaceforOrgAppFix = cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID );
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
Mutator<ByteBuffer> repairMutator = createMutator( cass.getApplicationKeyspace( MANAGEMENT_APPLICATION_ID ), be );
int index = 0;
- HashSet<UUID> orgUUIDs = getOrgUUIDS();
+ HashSet<UUID> orgUUIDs = new HashSet<UUID>( );
+ if(line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE )==null) {
+ orgUUIDs = getOrgUUIDS();
+ }
+ else{
+ orgUUIDs.add( UUID.fromString( line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE ) ) );
+ }
for ( UUID orgUUID : orgUUIDs ) {
@@ -167,7 +174,6 @@ public class ApplicationAuditOrgFix extends ToolBase {
logger.error( "There are no applications associated with this org: {}", orgUUID );
}
else {
- Boolean orgFixed = false;
for ( HColumn<DynamicComposite, ByteBuffer> col : cols ) {
DynamicComposite dynamicComposite = col.getName();
UUID applicationId = ( UUID ) dynamicComposite.get( 0 );
@@ -177,7 +183,25 @@ public class ApplicationAuditOrgFix extends ToolBase {
logger.info("The following application uuid: {} returns null for org uuid: {}",applicationId,orgUUID);
}
else{
- logger.info("The following application with information: {}, will be used to repair org uuid: {}.",applicationInfo,orgUUID);
+
+ String uniqueValue = applicationInfo.getName().split( "/" )[0];
+ logger.info("The following application with information: {} ,with the derived name: {} ,will be used to repair org uuid: {}",new Object[]{applicationInfo.getName(),uniqueValue,orgUUID});
+
+ Group organizationEntity = new Group();
+ organizationEntity.setPath( uniqueValue );
+ try {
+ em.create( orgUUID, Group.ENTITY_TYPE, organizationEntity.getProperties() );
+ organizationEntity = em.get( orgUUID, Group.class );
+ if ( organizationEntity == null ) {
+ logger.error( "OrgName: {} with uuid: {} could not be created", uniqueValue, orgUUID );
+ }
+ else{
+ logger.info( "OrgName: {} with uuid: {} was fixed.", uniqueValue, orgUUID );
+ }
+ }catch(Exception e){
+ logger.error("Could not create: {} because of {}",uniqueValue,e.getMessage());
+ }
+ break;
}
}
@@ -209,7 +233,7 @@ public class ApplicationAuditOrgFix extends ToolBase {
if(verificationOrganizationInfo==null){
logger.error("Repair failed for rowkey: {} and org uuid: {}",key,orgUUID);
}
- else if(verificationOrganizationInfo.getUuid().equals( organizationInfo.getUuid())&& verificationOrganizationInfo.equals( organizationInfo.getName() )){
+ else if(verificationOrganizationInfo.getUuid().equals( organizationInfo.getUuid())&& verificationOrganizationInfo.getName().equals( organizationInfo.getName() )){
logger.info( "Org name: {} with uuid is fixed: {}",organizationInfo.getUuid(),organizationInfo.getUuid() );
}
}