You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/11/05 01:34:51 UTC
usergrid git commit: Removing code that has been refactored out.
Edited flow to lead up to range query with more readability.
Repository: usergrid
Updated Branches:
refs/heads/USERGRID-1076 03a123dc8 -> 6e8b2b621
Removing code that has been refactored out. Edited flow to lead up to range query with more readability.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6e8b2b62
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6e8b2b62
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6e8b2b62
Branch: refs/heads/USERGRID-1076
Commit: 6e8b2b621cc45636bb322b03d0670d5d01b0a97d
Parents: 03a123d
Author: George Reyes <gr...@apache.org>
Authored: Wed Nov 4 16:34:48 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Wed Nov 4 16:34:48 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 206 +++++--------------
1 file changed, 54 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6e8b2b62/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 52288e4..9b1c202 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -98,12 +98,7 @@ import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
/**
* This is a utility to audit all available entity ids in the secondary index. It then checks to see if any index value
* is not present in the Entity_Index_Entries. If it is not, the value from the index is removed, and a forced re-index
- * is triggered
- * <p/>
- * USERGRID-323
- * <p/>
- * <p/>
- * UniqueIndexCleanup -app [appid] -col [collectionname]
+ * is triggered <p/> USERGRID-323 <p/> <p/> UniqueIndexCleanup -app [appid] -col [collectionname]
*
* @author tnine
*/
@@ -115,7 +110,6 @@ public class UniqueIndexCleanup extends ToolBase {
private static final int PAGE_SIZE = 100;
-
private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
/**
@@ -130,7 +124,7 @@ public class UniqueIndexCleanup extends ToolBase {
@Override
- @SuppressWarnings("static-access")
+ @SuppressWarnings( "static-access" )
public Options createOptions() {
@@ -169,97 +163,66 @@ public class UniqueIndexCleanup extends ToolBase {
public void runTool( CommandLine line ) throws Exception {
startSpring();
- logger.info( "Starting entity cleanup" );
-
- Map<String, UUID> apps = getApplications( emf, line );
-
-
- for ( Entry<String, UUID> app : apps.entrySet() ) {
-
- logger.info( "Starting cleanup for app {}", app.getKey() );
-
- UUID applicationId = app.getValue();
- EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
-
- //sanity check for corrupt apps
- Application appEntity = em.getApplication();
-
- if ( appEntity == null ) {
- logger.warn( "Application does not exist in data. {}", app.getKey() );
- continue;
- }
-
- CassandraService cass = em.getCass();
+ logger.info( "Starting entity unique index cleanup" );
- Keyspace ko = cass.getUsergridApplicationKeyspace();
- Mutator<ByteBuffer> m = createMutator( ko, be );
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
-
- // go through each collection and audit the values
- for ( String collectionName : getCollectionNames( em, line ) ) {
-
- RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery = HFactory
- .createRangeSlicesQuery( ko, be, be, be )
- .setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
- //not sure if I trust the lower two settings as it might iterfere with paging or set arbitrary limits and what I want to retrieve.
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
//That needs to be verified.
- .setKeys( null, null )
- .setRange( null, null, false, 100 );
-
+ .setKeys( null, null ).setRange( null, null, false, 100 );
- RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery,null,null );
- QueryResult<OrderedRows<ByteBuffer, ByteBuffer, ByteBuffer>> result = rangeSlicesQuery.execute();
- OrderedRows<ByteBuffer, ByteBuffer, ByteBuffer> rows = result.get();
- result.get().getList().get( 0 ).getColumnSlice();
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
- while(rangeSlicesIterator.hasNext()) {
- Row rangeSliceValue = rangeSlicesIterator.next();
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
- String returnedRowKey =
- new String( ( ( ByteBuffer ) rangeSliceValue.getKey() ).array(), Charsets.UTF_8 ).trim();
+ String returnedRowKey =
+ new String( ( ( ByteBuffer ) rangeSliceValue.getKey() ).array(), Charsets.UTF_8 ).trim();
- String[] parsedRowKey = returnedRowKey.split( ":" );
- if ( parsedRowKey[1].equals( "users" ) ) {
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+ UUID applicationId = UUID.fromString( parsedRowKey[0] );
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
- ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
- if ( columnSlice.getColumns().size() != 0 ) {
- System.out.println( returnedRowKey );
- List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- UUID entityId = ue.fromByteBuffer( col.getName() );
+ EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
+ Boolean cleanup = false;
- if(parsedRowKey[0].equals( MANAGEMENT_APPLICATION_ID.toString() )){
- if(managementService.getAdminUserByUuid( entityId )==null ){
- logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
- System.out.println( "Deleting column uuid: " + entityId.toString() );
+ //TODO: make parsed row key more human friendly. Anybody looking at it doesn't know what value means what.
+ if ( parsedRowKey[1].equals( "users" ) ) {
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ if ( columnSlice.getColumns().size() != 0 ) {
+ System.out.println( returnedRowKey );
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
- Object key = key( applicationId, collectionName, parsedRowKey[2], parsedRowKey[3]);
- addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
- m.execute();
- continue;
- }
- }
- else if ( em.get( entityId ) == null ) {
- logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
- System.out.println( "Deleting column uuid: " + entityId.toString() );
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID entityId = ue.fromByteBuffer( col.getName() );
- Object key = key( applicationId, collectionName, parsedRowKey[2], parsedRowKey[3]);
- addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
- m.execute();
- continue;
- }
+ if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
+ if ( managementService.getAdminUserByUuid( entityId ) == null ) {
+ cleanup = true;
}
}
+ else if ( em.get( entityId ) == null ) {
+ cleanup =true;
+ }
+
+ if(cleanup == true){
+ DeleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue,
+ entityId );
+ cleanup = false;
+ }
}
}
-
}
}
@@ -267,78 +230,17 @@ public class UniqueIndexCleanup extends ToolBase {
}
- private Map<String, UUID> getApplications( EntityManagerFactory emf, CommandLine line ) throws Exception {
- String appName = line.getOptionValue( APPLICATION_ARG );
-
- if ( appName == null ) {
- return emf.getApplications();
- }
-
- ApplicationInfo app = managementService.getApplicationInfo( Identifier.from( appName ) );
-
- if ( app == null ) {
- logger.error( "Could not find application with id or name {}", appName );
- System.exit( 3 );
- }
-
-
- Map<String, UUID> apps = new HashMap<String, UUID>();
-
- apps.put( app.getName(), app.getId() );
-
- return apps;
- }
-
-
- private Set<String> getCollectionNames( EntityManager em, CommandLine line ) throws Exception {
-
- String collectionName = line.getOptionValue( COLLECTION_ARG );
-
- if ( collectionName == null ) {
- return em.getApplicationCollections();
- }
-
-
- Set<String> names = new HashSet<String>();
- names.add( collectionName );
-
- return names;
- }
-
-
- private List<HColumn<ByteBuffer, ByteBuffer>> scanIndexForAllTypes( Keyspace ko,
- IndexBucketLocator indexBucketLocator,
- UUID applicationId, Object rowKey,
- UUID entityId, String prop ) throws Exception {
-
- //TODO Determine the index bucket. Scan the entire index for properties with this entityId.
-
-
- DynamicComposite start = null;
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols;
-
- List<HColumn<ByteBuffer, ByteBuffer>> results = new ArrayList<HColumn<ByteBuffer, ByteBuffer>>();
-
-
- do {
- cols = cass.getColumns( ko, ENTITY_INDEX, rowKey, start, null, 100, false );
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( col.getName().duplicate() );
-
- UUID storedId = ( UUID ) secondaryIndexValue.get( 2 );
-
- //add it to the set. We can't short circuit due to property ordering
- if ( entityId.equals( storedId ) ) {
- results.add( col );
- }
-
- start = secondaryIndexValue;
- }
- }
- while ( cols.size() == 100 );
+ private void DeleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId,
+ final String collectionName, final String uniqueValueKey, final String uniqueValue,
+ final UUID entityId ) throws Exception {
+ logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
+ System.out.println( "Deleting column uuid: " + entityId.toString() );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
- return results;
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
+ m.execute();
+ return;
}
}