You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/11/18 22:21:36 UTC
[1/2] usergrid git commit: Fixed array issue in read repair that
would break after a certain size of columns. Using the read repair for
UserUniqueIndexCleanup instead of homegrown option. FIXED 401'S FOR TO BE NOT
FOUND LIKE THEY SHOULD HAVE BEEN.
Repository: usergrid
Updated Branches:
refs/heads/1.x 9f3299ef6 -> 65ed997c1
Fixed array issue in read repair that would break after a certain size of columns.
Using the read repair for UserUniqueIndexCleanup instead of homegrown option.
FIXED 401'S FOR TO BE NOT FOUND LIKE THEY SHOULD HAVE BEEN.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cc210463
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cc210463
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cc210463
Branch: refs/heads/1.x
Commit: cc2104638f6fc5c733197330869569b9ac4397f9
Parents: d2bd3e8
Author: George Reyes <gr...@apache.org>
Authored: Wed Nov 18 10:12:42 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Wed Nov 18 10:12:42 2015 -0800
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 6 +-
.../ServiceResourceNotFoundExceptionMapper.java | 7 +-
.../usergrid/tools/UserUniqueIndexCleanup.java | 154 ++++++-------------
3 files changed, 53 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc210463/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 38a80a0..ec99d53 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -550,10 +550,11 @@ public class EntityManagerImpl implements EntityManager {
//check to see if the single value is valid. If it is not valid then it will be zero and go through
//the code below.
if(cols.size() == 1){
- logger.info("Verifying that column is still valid, if not will be removed");
+ logger.debug("Verifying that column is still valid, if not will be removed");
UUID indexCorruptionUuid = ue.fromByteBuffer( cols.get( 0 ).getName());
if (get(indexCorruptionUuid) == null ) {
+ logger.debug( "Deleting the following uuid: {} from the application: {}",indexCorruptionUuid,applicationId );
deleteUniqueColumn( ownerEntityId, key, indexCorruptionUuid );
cols.remove( 0 );
}
@@ -577,7 +578,7 @@ public class EntityManagerImpl implements EntityManager {
//Contains entities that weren't deleted but are still in index.
//maybe use a set or list so you don't have to keep track of an index.
- Entity[] entities = new Entity[cols.size()];
+ Entity[] entities = new Entity[indexingColumns.size()];
int index = 0;
for ( HColumn<ByteBuffer, ByteBuffer> col : indexingColumns ) {
@@ -586,6 +587,7 @@ public class EntityManagerImpl implements EntityManager {
entities[index] = get(indexCorruptionUuid);
if (entities[index] == null ) {
+ logger.debug("deleting uuid: {} from {} because it no longer exists.",indexCorruptionUuid,ownerEntityId);
deleteUniqueColumn( ownerEntityId, key, indexCorruptionUuid );
}
else{
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc210463/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ServiceResourceNotFoundExceptionMapper.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ServiceResourceNotFoundExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ServiceResourceNotFoundExceptionMapper.java
index 64eb5d9..7cfd8e6 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ServiceResourceNotFoundExceptionMapper.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/ServiceResourceNotFoundExceptionMapper.java
@@ -32,11 +32,6 @@ public class ServiceResourceNotFoundExceptionMapper extends AbstractExceptionMap
@Override
public Response toResponse( ServiceResourceNotFoundException e ) {
- if ( SubjectUtils.getSubjectUserId() == null ) {
- return toResponse( UNAUTHORIZED, e );
- }
- else {
- return toResponse( NOT_FOUND, e );
- }
+ return toResponse( NOT_FOUND, e );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc210463/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
index aec4266..d9d0bb5 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
@@ -19,7 +19,9 @@ package org.apache.usergrid.tools;
import java.nio.ByteBuffer;
import java.nio.charset.Charset;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import org.slf4j.Logger;
@@ -33,6 +35,7 @@ import org.apache.commons.cli.Options;
import org.apache.thrift.TBaseHelper;
import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
import org.apache.usergrid.utils.UUIDUtils;
@@ -55,19 +58,22 @@ import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+
/**
+ * This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users then it
+ * deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row. If
+ * there exists more than one existing column then the one with the most recent timestamp wins and the other is
+ * deleted.
*
- *This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users
- * then it deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row.
- * If there exists more than one existing column then the one with the most recent timestamp wins and the other is deleted.
- *
- *If you want the run the tool on their cluster the following is what you need to do
- * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host <cassandra_host_here> > log.txt
+ * If you want the run the tool on their cluster the following is what you need to do nohup java
+ * -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> > log.txt
*
* if there is a specific value you want to run the tool on then you need the following
-
- * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host <cassandra_host_here>
- * -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value <unique_property_value> > log.txt
+ *
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host
+ * <cassandra_host_here> -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value
+ * <unique_property_value> > log.txt
*
* @author grey
*/
@@ -152,7 +158,7 @@ public class UserUniqueIndexCleanup extends ToolBase {
deleteInvalidValuesForUniqueProperty( m, line );
}
else {
-//maybe put a byte buffer infront.
+ //maybe put a byte buffer infront.
RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
//not sure if I trust the lower two settings as it might iterfere with paging or set
@@ -185,13 +191,13 @@ public class UserUniqueIndexCleanup extends ToolBase {
parsedRowKey = garbageRowKeyParser( parsedRowKey );
if ( parsedRowKey == null ) {
- logger.error( "{} is a invalid row key, and unparseable. Skipped...",returnedRowKey );
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
continue;
}
}
//if the rowkey contains less than four parts then it is completely invalid
else if ( parsedRowKey.length < 4 ) {
- logger.error( "{} is a invalid row key, and unparseable. Skipped...",returnedRowKey );
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...", returnedRowKey );
continue;
}
@@ -200,7 +206,7 @@ public class UserUniqueIndexCleanup extends ToolBase {
applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
}
catch ( Exception e ) {
- logger.error( "could not parse {} despite earlier parsing. Skipping...",parsedRowKey[0] );
+ logger.error( "could not parse {} despite earlier parsing. Skipping...", parsedRowKey[0] );
continue;
}
String collectionName = parsedRowKey[1];
@@ -217,9 +223,10 @@ public class UserUniqueIndexCleanup extends ToolBase {
deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
}
else {
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,returnedRowKey );
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,
+ returnedRowKey );
}
- // }
+ // }
}
}
}
@@ -266,7 +273,7 @@ public class UserUniqueIndexCleanup extends ToolBase {
private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
final String uniqueValueKey, final String uniqueValue ) throws Exception {
- logger.debug( "Found 0 uuid's associated with {} Deleting row.",uniqueValue );
+ logger.debug( "Found 0 uuid's associated with {} Deleting row.", uniqueValue );
UUID timestampUuid = newTimeUUID();
long timestamp = getTimestampInMicros( timestampUuid );
@@ -289,111 +296,47 @@ public class UserUniqueIndexCleanup extends ToolBase {
//these columns all come from the same row key, which means they each belong to the same row key identifier
//thus mixing and matching them in the below if cases won't matter.
Entity[] entities = new Entity[cols.size()];
- int numberOfRetrys = 5;
+ int numberOfRetrys = 8;
int numberOfTimesRetrying = 0;
int index = 0;
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- UUID entityId = ue.fromByteBuffer( col.getName() );
- //could be the same and just do a entity get
- if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
- for(numberOfTimesRetrying = 0; numberOfTimesRetrying<numberOfRetrys; numberOfTimesRetrying++){
- try {
- entities[index] = managementService.getAdminUserEntityByUuid( entityId );
- break;
- }catch(TimedOutException toe){
- Thread.sleep( 2000 );
- }
- }
- if(numberOfTimesRetrying == numberOfRetrys ){
- logger.error( "Tried {} number of times to get the following uuid: {} but failed.Moving on",numberOfRetrys,entityId );
- continue;
- }
-
- if ( entities[index] == null ) {
- cleanup = true;
+ for ( int i = 0; i < numberOfRetrys; i++ ) {
+ try {
+ Map<String, EntityRef> results =
+ em.getAlias( applicationId, collectionName, Collections.singletonList( uniqueValue ) );
+ if ( results.size() > 1 ) {
+ logger.error("failed to clean up {} from application {}. Please clean manually.",uniqueValue,applicationId);
+ break;
}
else {
- index++;
- }
- }
- else {
-
- for(int i = 0; i<numberOfRetrys; i++){
- try {
- entities[index] = em.get( entityId );
- break;
- }catch(TimedOutException toe){
- Thread.sleep( 2000 );
- }
- }
- if(numberOfTimesRetrying == numberOfRetrys ){
- logger.error( "Tried {} number of times to get the following uuid: {} but failed.Moving on",numberOfRetrys,entityId );
continue;
}
-
- if ( entities[index] == null ) {
- cleanup = true;
- }
- else {
- index++;
- }
}
-
- if ( cleanup == true ) {
- numberOfColumnsDeleted++;
- deleteUniqueValue(applicationId, collectionName, uniqueValueKey, uniqueValue, entityId );
- cleanup = false;
+ catch ( Exception toe ) {
+ logger.error( "timeout doing em getAlias repair. This is the {} number of repairs attempted", i );
+ toe.printStackTrace();
+ Thread.sleep( 1000 * i );
}
}
-
- //this means that the same unique rowkey has two values associated with it
- if(index>2){
- Entity mostRecentEntity = entities[0];
- for(Entity entity: entities){
-
- mostRecentEntity = verifyModifiedTimestamp( mostRecentEntity );
- entity = verifyModifiedTimestamp( entity );
-
-
- if(mostRecentEntity.getModified() > entity.getModified()){
- logger.error( "Deleting {} because it is the older column in the following rowkey: {} ",entity.getUuid().toString(),rowKey );
- em.deleteEntity( entity.getUuid() );
- }
- else if (mostRecentEntity.getModified() < entity.getModified()){
- logger.error( "Deleting {} because it is the older column in the following rowkey: {} ",mostRecentEntity.getUuid().toString(),rowKey );
- em.deleteEntity( mostRecentEntity.getUuid() );
- mostRecentEntity = entity;
- }
- else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
- logger.error( "Entities with unique value: {} contains two or more entities with the same modified time. Please manually fix these"
- + "entities by deleting one.",rowKey);
- }
- }
- }
-
-
- //a safer way to do this would be to try to do another get and verify there is nothing left in the column
- //instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
- if ( cols.size() == numberOfColumnsDeleted ) {
- deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
- }
}
private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
Entity entity = unverifiedEntity;
- if(entity !=null && entity.getModified()==null) {
- if(entity.getCreated()!=null){
- logger.debug("{} has no modified. Subsituting created timestamp for their modified timestamp.Manually adding one for comparison purposes",entity.getUuid());
+ if ( entity != null && entity.getModified() == null ) {
+ if ( entity.getCreated() != null ) {
+ logger.debug(
+ "{} has no modified. Subsituting created timestamp for their modified timestamp.Manually "
+ + "adding one for comparison purposes",
+ entity.getUuid() );
entity.setModified( entity.getCreated() );
return entity;
}
- else{
+ else {
logger.error( "Found no created or modified timestamp. Please remake the following entity: {}."
- + " Setting both created and modified to 1",entity.getUuid().toString() );
+ + " Setting both created and modified to 1", entity.getUuid().toString() );
entity.setCreated( 1L );
entity.setModified( 1L );
return entity;
@@ -421,10 +364,10 @@ public class UserUniqueIndexCleanup extends ToolBase {
if ( cols.size() == 0 ) {
- logger.error( "This row key: {} has zero columns. Deleting...",key.toString() );
+ logger.error( "This row key: {} has zero columns. Deleting...", key.toString() );
}
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,key.toString() );
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols, key.toString() );
}
@@ -436,7 +379,7 @@ public class UserUniqueIndexCleanup extends ToolBase {
stringToBeTruncated = stringToBeTruncated.substring( index );
}
else {
- logger.error( "{} is unparsable",garbageString );
+ logger.error( "{} is unparsable", garbageString );
break;
}
}
@@ -459,9 +402,8 @@ public class UserUniqueIndexCleanup extends ToolBase {
}
- private void deleteUniqueValue( final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue, final UUID entityId )
- throws Exception {
+ private void deleteUniqueValue( final UUID applicationId, final String collectionName, final String uniqueValueKey,
+ final String uniqueValue, final UUID entityId ) throws Exception {
logger.warn( "Entity with id {} did not exist in app {} Deleting", entityId, applicationId );
UUID timestampUuid = newTimeUUID();
long timestamp = getTimestampInMicros( timestampUuid );
[2/2] usergrid git commit: This closes #440
Posted by sn...@apache.org.
This closes #440
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/65ed997c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/65ed997c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/65ed997c
Branch: refs/heads/1.x
Commit: 65ed997c1f2977124870e5e788973361d2e81930
Parents: 9f3299e cc21046
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Nov 18 16:21:11 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Nov 18 16:21:11 2015 -0500
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 6 +-
.../ServiceResourceNotFoundExceptionMapper.java | 7 +-
.../usergrid/tools/UserUniqueIndexCleanup.java | 154 ++++++-------------
3 files changed, 53 insertions(+), 114 deletions(-)
----------------------------------------------------------------------