You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/11/17 23:04:56 UTC
[01/39] usergrid git commit: Added the collection tool that needs to
be expanded
Repository: usergrid
Updated Branches:
refs/heads/1.x bd4fd2643 -> 9f3299ef6
Added the collection tool that needs to be expanded
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a72da10f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a72da10f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a72da10f
Branch: refs/heads/1.x
Commit: a72da10f85bd01b47950e18b4ab2f47dfa4ecf23
Parents: afe0c51
Author: George Reyes <gr...@apache.org>
Authored: Wed Oct 28 11:23:37 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Wed Oct 28 11:23:37 2015 -0700
----------------------------------------------------------------------
.../rest/applications/users/UsersResource.java | 3 +
...cateUniquePropertyExistsExceptionMapper.java | 6 +
.../usergrid/tools/CollectionUserFix.java | 255 +++++++++++++++++++
3 files changed, 264 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
index 6325d5a..408299a 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
@@ -100,6 +100,9 @@ public class UsersResource extends ServiceResource {
@Path("{itemName}")
public AbstractContextResource addNameParameter( @Context UriInfo ui, @PathParam("itemName") PathSegment itemName )
throws Exception {
+ //TODO: read repair if entity ref doesn't match returned entity. Detect index corruption and delete reference if
+ //reference isn't correct.
+
logger.info( "ServiceResource.addNameParameter" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
index b910b58..0ea1567 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
@@ -29,6 +29,12 @@ import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
public class DuplicateUniquePropertyExistsExceptionMapper
extends AbstractExceptionMapper<DuplicateUniquePropertyExistsException> {
+ //when you get this exception then fire a repair task that will grab a thread from a thread pool and async run the repair task
+ // That task will then go through and double checkk all the entities and verify that their ref's and entities match up.
+
+
+ //npe, delete entity ref
+ //mismatched pointers, update the entity ref with whatever is in the collection. ( retrieve via query ) .
@Override
public Response toResponse( DuplicateUniquePropertyExistsException e ) {
return toResponse( BAD_REQUEST, e );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
new file mode 100644
index 0000000..829487a
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Identifier;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.Application;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+
+/**
+ * Fixes issues where admin data is in organizations but can't accessed or reset the password of. This usually returns
+ * the error "Could not find organization for email" because the user can't be found but when the org is queried the
+ * user shows up there. If you see that then run this tool.
+ */
+public class CollectionUserFix extends ExportingToolBase {
+ private static final int PAGE_SIZE = 100;
+
+ private static final Logger logger = LoggerFactory.getLogger( CollectionUserFix.class );
+
+ /**
+ *
+ */
+ private static final String ORGANIZATION_ARG = "org";
+
+ /**
+ *
+ */
+ private static final String APPLICATION_ARG = "app";
+
+ /**
+ *
+ */
+ private static final String QUERY_ARG = "query";
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ Option orgOption = OptionBuilder.withArgName( ORGANIZATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "organization id" ).create( ORGANIZATION_ARG );
+ Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "application uuid" ).create( APPLICATION_ARG );
+ Option queryOption = OptionBuilder.withArgName( QUERY_ARG ).hasArg().isRequired( false )
+ .withDescription( "query" ).create( QUERY_ARG );
+
+ //add a query to it
+
+ Options options = new Options();
+ options.addOption( hostOption );
+ options.addOption( orgOption );
+ options.addOption( appOption );
+ options.addOption( queryOption );
+
+ return options;
+ }
+
+
+ /**
+ * psudeo code
+ * if org id is present then ignore the application id
+ * go through everysingle application/users entity. And check to see if it can be queried by username.
+ * else if app id is present
+ * go through the applications/users
+ *
+ *
+ * Take the list of applications to go through and go through them one by one. ( in the latter else case we will only
+ *
+ * @param line
+ * @throws Exception
+ */
+
+
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+
+
+ startSpring();
+
+ logger.info( "Starting crawl of all users" );
+ System.out.println( "Starting crawl of all users" );
+ Set<Application> applicationSet = new HashSet<Application>( );
+ EntityManager em = null;
+
+
+
+ if(line.getOptionValue( ORGANIZATION_ARG )==null||line.getOptionValue( ORGANIZATION_ARG ).isEmpty()) {
+ em = emf.getEntityManager( UUID.fromString( line.getOptionValue( APPLICATION_ARG ) ) );
+ applicationSet.add( em.getApplication() );
+ }
+ else{
+ BiMap applicationsForOrganization =
+ managementService
+ .getApplicationsForOrganization( UUID.fromString(line.getOptionValue( ORGANIZATION_ARG )));
+
+ applicationSet = applicationsForOrganization.keySet();
+ }
+ startCollectionFlow( em, applicationSet, line.getOptionValue( QUERY_ARG ) );
+
+
+ logger.info( "Repair complete" );
+ System.out.println("Repair Complete");
+ }
+
+
+ //make
+ private void startCollectionFlow(final EntityManager entityManager, final Set<Application> app, final String queryString )
+ throws Exception {// search for all orgs
+
+ Query query = new Query();
+ if(queryString!=null){
+ query = query.fromQL( queryString );
+ }
+ query.setLimit( PAGE_SIZE );
+ Results r = null;
+ EntityManager em = null;
+
+ for ( Application application : app ) {
+ //This will hold all of the applications users. This will be stored in memory to do a simple check to see if
+ //there are any duped usernames in the collection.
+ //Memory concerns means that
+ Multimap<String, UUID> usernames = HashMultimap.create();
+
+ //This means that we need to set it for each and every single application thus it gets set here instead of
+ //the method that calls us.
+ if(entityManager == null){
+ em = emf.getEntityManager( application.getUuid() );
+ }
+ else {
+ em = entityManager;
+ }
+
+ do {
+
+ //get all users in the management app and page for each set of a PAGE_SIZE
+ r = em.searchCollection( application, "users", query );
+ System.out.println("found "+r.size()+" number of entities");
+
+ for ( Entity entity : r.getEntities() ) {
+ //grab all usernames returned.
+ usernames.put( entity.getProperty( "username" ).toString().toLowerCase(), entity.getUuid() );
+ }
+
+ query.setCursor( r.getCursor() );
+
+ logger.info( "Searching next page" );
+ System.out.println("Searching next page");
+ }
+ while ( r != null && r.size() == PAGE_SIZE );
+
+ System.out.println("Starting username crawl of "+usernames.size()+" number of usernames");
+ //do a get on a specific username, if it shows up more than once then remove it
+ for ( String username : usernames.keySet() ) {
+ Collection<UUID> ids = usernames.get( username );
+
+ if ( ids.size() > 1 ) {
+ logger.info( "Found multiple users with the username {}", username );
+ System.out.println( "Found multiple users with the username: " + username );
+ }
+
+ //UserInfo targetUser = managementService.getAdminUserByEmail( email );
+ Identifier identifier = new Identifier();
+ EntityRef targetUser = em.getUserByIdentifier( identifier.fromName( username ) );
+
+
+ if ( targetUser == null ) {
+ //This means that the username isn't properly associated with targetUser
+ List<UUID> tempIds = new ArrayList<UUID>( ids );
+ //Collections.sort( tempIds );
+
+ UUID toLoad = tempIds.get( 0 );
+
+ logger.warn( "Could not load target user by username {}, loading by UUID {} instead", username, toLoad );
+ System.out.println( "Could not load the target user by username: " + username
+ + ". Loading by the following uuid instead: " + toLoad.toString() );
+
+ User targetUserEntity = null;
+ try {
+ targetUserEntity = em.get( toLoad, User.class );
+ }catch(Exception e){
+ System.out.println("The follow uuid has no data in this cassandra node: "+toLoad.toString());
+ throw e;
+ }
+
+
+ try {
+ if ( targetUserEntity != null&& targetUserEntity.getUuid().equals( toLoad )) {
+ System.out.println("Updating uuid: "+targetUserEntity.getUuid().toString());
+ em.update( targetUserEntity );
+ }
+ }
+ catch ( DuplicateUniquePropertyExistsException dup ) {
+ System.out.println( "Found duplicate unique property: " + dup.getPropertyName() + ". "
+ + "Duplicate property is: "
+ + dup.getPropertyValue() );
+ //if there are duplicate unique properties then
+ if ( dup.getPropertyName().equals( "username" ) ) {
+ System.out.println("can I replace this with a different value since these are duplicated in the code base");
+ //targetUserEntity.setUsername( targetUserEntity.getUsername() );
+ }
+ //else throw dup;
+ }
+ catch (Exception e){
+ System.out.println("There was an issue with updating: "+e.getMessage());
+ }
+ }
+ }
+ }
+ }
+}
[35/39] usergrid git commit: Added instructions on how to
runUserUniqueIndexCleanup. Replaced UniqueIndexCleanup with
UserUniqueIndexCleanup as it only cleans up users. Changed all
system.out.prints to logger type messages
Posted by sn...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
new file mode 100644
index 0000000..e08b579
--- /dev/null
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
@@ -0,0 +1,490 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.junit.*;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.RandomStringUtils;
+
+import org.apache.usergrid.ServiceITSetup;
+import org.apache.usergrid.ServiceITSetupImpl;
+import org.apache.usergrid.ServiceITSuite;
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationOwnerInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.mutation.Mutator;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+
+/**
+ * Created by ApigeeCorporation on 11/2/15.
+ */
+public class UserUniqueIndexCleanupTest {
+ static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
+
+ int NUM_COLLECTIONS = 10;
+ int NUM_ENTITIES = 50;
+ int NUM_CONNECTIONS = 3;
+
+ @ClassRule
+ public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
+
+
+ @org.junit.Test
+ public void testBasicOperation() throws Exception {
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:9160"
+ }, false );
+
+ System.out.println( "completed" );
+ }
+
+ //this test is perfect for the other tool the userCollectionFix tool as this is what I believe they were seeing.
+ @Ignore ("WRong test not made for unique index cleanup.")
+ public void testRepairOfSingleEntityMissingColumnWrongTool() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,username,username,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+ CassandraService cass = setup.getCassSvc();
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationInfo.getId() ), ENTITY_UNIQUE, key, null, null,
+ 2, false );
+
+ Set<UUID> results = new HashSet<UUID>( cols.size() );
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ results.add( ue.fromByteBuffer( col.getName() ) );
+ }
+
+ UUID uuid = results.iterator().next();
+
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+
+ //Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ key = key( applicationInfo.getId(), collectionName, "username", username );
+ //addDeleteToMutator( m, ENTITY_UNIQUE, key, uuid, timestamp );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, uuid );
+ m.execute();
+
+ assertNull( entityManager.getAlias( applicationInfo.getId(), collectionName, username ) );
+
+ assertNotNull(entityManager.get( entityToBeCorrupted.getUuid() ));
+
+ //run the cleanup
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
+
+
+ //here you need to add a delete to the mutator then recheck it and see if the entity is the same as .
+ Thread.sleep( 2000 );
+ assertNull( entityManager.get( entityToBeCorrupted.getUuid() ) );
+
+ //When you come back you also need to emulate the tests to delete what is out of the uuid without any corresponding data.
+ //maybe it'll be easier to just do an insert into the EntityUnique row without doint it into any other row and
+ //then verify the data like that. Then you don't have to do deletes out of other things.
+
+ }
+
+ //For this test you need to insert a dummy key with a dummy column that leads to nowhere
+ //then run the unique index cleanup.
+ //checks for bug when only column doesn't exist make sure to delete the row as well.
+
+ //due to the read repair this is no longer a valid test of hte unique index cleanup
+ @Ignore
+ public void testRepairOfSingleEntity() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ //assertNull(entityManager.get( testEntityUUID ));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ //The Get above should have repaired the entity allowing it run
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ //fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+ fail();
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+
+ //run the cleanup
+// UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+// uniqueIndexCleanup.startTool( new String[] {
+// "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+// }, false );
+
+
+ entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ assertNotNull( entityToBeCorrupted );
+ assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername());
+
+ }
+
+ //For this test you need to insert a dummy key with a dummy column that leads to nowhere
+ //then run the unique index cleanup.
+ //checks for bug when only column doesn't exist make sure to delete the row as well.
+ // Due to the read repair this is no longer a valid test of unique index cleanup.
+ @Test
+ @Ignore
+ public void testRepairOfMultipleEntities() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ int numberOfEntitiesToCreate = 1000;
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ String[] usernames = new String[numberOfEntitiesToCreate];
+
+ int index = 0;
+ while(index < numberOfEntitiesToCreate) {
+
+ usernames[index]=username+index;
+
+ Map<String, Object> userInfo = new HashMap<String, Object>();
+ userInfo.put( "username", usernames[index] );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", usernames[index] );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name
+ addInsertToMutator( m, ENTITY_UNIQUE, key, testEntityUUID, 0, createTimestamp() );
+
+ m.execute();
+ index++;
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+ }
+
+ //run the cleanup
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
+
+ for(String user:usernames ) {
+ Map<String, Object> userInfo = new HashMap<String, Object>();
+ userInfo.put( "username", user);
+ Entity entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+
+ assertNotNull( entityToBeCorrupted );
+ assertEquals( user, ( ( User ) entityToBeCorrupted ).getUsername() );
+ }
+ }
+
+ @Test
+ public void testRepairOfOnlyOneOfTwoColumns() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ //create a new column
+ Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+
+ //run the cleanup
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
+
+ //verifies it works now.
+ assertNotNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+ }
+
+ @Test
+ public void testStringParsing(){
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+
+ //String garbageString = ")xƐ:�^Q?�I�p\\�/2178c690-3a6f-11e4-aec6-48fa705cb26f:users:username:test";
+
+ UUID uuid = UUIDUtils.newTimeUUID();
+
+ String garbageString = "S2^? >-^Q��%\"�]^S:"+uuid+":users:username:2";
+
+ String[] parsedString = garbageString.split( ":" );
+
+ String[] cleanedString = userUniqueIndexCleanup.garbageRowKeyParser( parsedString );
+
+ assertEquals( uuid.toString(),cleanedString[0] );
+ assertEquals( "users",cleanedString[1] );
+ assertEquals( "username",cleanedString[2] );
+ assertEquals( "2",cleanedString[3] );
+ }
+
+ //POinting at single values is broken now but not entirely used right now anyways.
+ //@Ignore
+ @Test
+ public void testRepairOfOnlyOneOfTwoColumnsWhilePointingAtSingleValue() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ //create a new column
+ Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e) {
+ fail( "shouldn't throw something else i think" );
+ }
+
+ //NEED TO FAIL MORE GRACEFULLY
+ //run the cleanup
+ UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
+ userUniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort(),
+ "-col",collectionName,
+ "-app",applicationInfo.getId().toString(),
+ "-property","username",
+ "-value",username
+ }, false );
+
+ //verifies it works now.
+ assertNotNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+ }
+}
+
[20/39] usergrid git commit: Added tests and fixed bug where the tool
could not handle non standard row keys from older versions of usergrid.
Posted by sn...@apache.org.
Added tests and fixed bug where the tool could not handle non standard row keys from older versions of usergrid.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0ba197e4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0ba197e4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0ba197e4
Branch: refs/heads/1.x
Commit: 0ba197e434ca6cc146c8c36e53f692bed124b868
Parents: 14eec4f
Author: George Reyes <gr...@apache.org>
Authored: Mon Nov 9 15:10:13 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Mon Nov 9 15:10:13 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 114 ++++++++++++++++---
.../usergrid/tools/UniqueIndexCleanupTest.java | 33 +++++-
2 files changed, 126 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0ba197e4/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index e1de85a..fe3a74e 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -32,6 +32,7 @@ import org.apache.commons.cli.Options;
import org.apache.thrift.TBaseHelper;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.utils.StringUtils;
import org.apache.usergrid.utils.UUIDUtils;
import me.prettyprint.cassandra.service.RangeSlicesIterator;
@@ -164,24 +165,53 @@ public class UniqueIndexCleanup extends ToolBase {
String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
Charset.defaultCharset() ).trim();
- String[] parsedRowKey = returnedRowKey.split( ":" );
- UUID applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
- String collectionName = parsedRowKey[1];
- String uniqueValueKey = parsedRowKey[2];
- String uniqueValue = parsedRowKey[3];
- if ( collectionName.equals( "users" ) ) {
- ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
- if ( columnSlice.getColumns().size() != 0 ) {
- System.out.println( returnedRowKey );
- List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
- if ( cols.size() == 0 ) {
- deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+ //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ if(returnedRowKey.contains( "users" )) {
+
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+
+ //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
+ if ( parsedRowKey.length > 4 ) {
+ parsedRowKey = garbageRowKeyParser(parsedRowKey);
+
+ if(parsedRowKey == null) {
+ System.out.println( returnedRowKey + " is a invalid row key, and unparseable. Skipped..." );
+ continue;
}
- else {
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols );
+ }
+ //if the rowkey contains less than four parts then it is completely invalid
+ else if ( parsedRowKey.length < 4){
+ System.out.println( returnedRowKey + " is a invalid row key and will be skipped" );
+ continue;
+ }
+
+ UUID applicationId = null;
+ try {
+ applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ }
+ catch ( Exception e ) {
+ continue;
+ }
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
+
+
+ if ( collectionName.equals( "users" ) ) {
+
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ if ( columnSlice.getColumns().size() != 0 ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+ if ( cols.size() == 0 ) {
+ deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
+ }
+ else {
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols );
+ }
}
}
}
@@ -191,6 +221,42 @@ public class UniqueIndexCleanup extends ToolBase {
logger.info( "Completed audit of apps" );
}
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser(String[] parsedRowKey){
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while(modifiedRowKey!=null) {
+ if(modifiedRowKey.length < 4){
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for( int i = 1; i < modifiedRowKey.length; i++){
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if(i+1 != modifiedRowKey.length){
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
final String uniqueValueKey, final String uniqueValue ) throws Exception {
@@ -234,6 +300,8 @@ public class UniqueIndexCleanup extends ToolBase {
cleanup = false;
}
}
+ //a safer way to do this would be to try to do another get and verify there is nothing left in the column
+ //instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
if(cols.size()==numberOfColumnsDeleted){
deleteRow( m,applicationId,collectionName,uniqueValueKey,uniqueValue );
}
@@ -251,9 +319,11 @@ public class UniqueIndexCleanup extends ToolBase {
List<HColumn<ByteBuffer, ByteBuffer>> cols =
- cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
+ cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
false );
+
+
if ( cols.size() == 0 ) {
System.out.println("Zero entities were found for this unique value. Its possible it doesn't exist or you typed in in wrong :p.");
}
@@ -277,6 +347,20 @@ public class UniqueIndexCleanup extends ToolBase {
return stringToBeTruncated;
}
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
private void deleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
final String uniqueValueKey, final String uniqueValue, final UUID entityId )
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0ba197e4/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
index 315cc16..664ca7c 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -113,7 +113,7 @@ public class UniqueIndexCleanupTest {
}
//this test is perfect for the other tool the userCollectionFix tool as this is what I believe they were seeing.
- @Test
+ @Ignore ("WRong test not made for unique index cleanup.")
public void testRepairOfSingleEntityMissingColumnWrongTool() throws Exception{
String rand = RandomStringUtils.randomAlphanumeric( 10 );
@@ -175,7 +175,7 @@ public class UniqueIndexCleanupTest {
}, false );
- //here you need to add a delete to the mutator then recheck it and see if the entity is the same as millicoms.
+ //here you need to add a delete to the mutator then recheck it and see if the entity is the same as .
Thread.sleep( 2000 );
assertNull( entityManager.get( entityToBeCorrupted.getUuid() ) );
@@ -407,8 +407,8 @@ public class UniqueIndexCleanupTest {
//should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
//TODO: fix the below so that it fails everytime.
//50/50 chance to succeed or fail
- assertNull( entityManager
- .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+// assertNull( entityManager
+// .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
//run the cleanup
@@ -424,6 +424,27 @@ public class UniqueIndexCleanupTest {
}
@Test
+ public void testStringParsing(){
+ UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+
+ //String garbageString = ")xƐ:�^Q��I�p\\�/2178c690-3a6f-11e4-aec6-48fa705cb26f:users:username:test";
+
+ UUID uuid = UUIDUtils.newTimeUUID();
+
+ String garbageString = "S2^? <-^Q��%\"�]^S:"+uuid+":users:username:2";
+
+ String[] parsedString = garbageString.split( ":" );
+
+ String[] cleanedString = uniqueIndexCleanup.garbageRowKeyParser( parsedString );
+
+ assertEquals( uuid.toString(),cleanedString[0] );
+ assertEquals( "users",cleanedString[1] );
+ assertEquals( "username",cleanedString[2] );
+ assertEquals( "2",cleanedString[3] );
+ }
+
+ //POinting at single values is broken now but not entirely used right now anyways.
+ @Test
public void testRepairOfOnlyOneOfTwoColumnsWhilePointingAtSingleValue() throws Exception{
String rand = RandomStringUtils.randomAlphanumeric( 10 );
@@ -484,8 +505,8 @@ public class UniqueIndexCleanupTest {
}
//should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
- assertNull( entityManager
- .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+// assertNull( entityManager
+// .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
//run the cleanup
[26/39] usergrid git commit: Added fix for rows that contain
duplicate valid existing columns.
Posted by sn...@apache.org.
Added fix for rows that contain duplicate valid existing columns.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3d5ae464
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3d5ae464
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3d5ae464
Branch: refs/heads/1.x
Commit: 3d5ae464766a12b3fac1d82b67da667ef28d17fd
Parents: 00171f0
Author: George Reyes <gr...@apache.org>
Authored: Tue Nov 10 14:49:40 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Tue Nov 10 14:49:40 2015 -0800
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 52 +++++++++++++++-----
1 file changed, 41 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3d5ae464/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index ecd0f0a..76cb6d7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -559,27 +559,45 @@ public class EntityManagerImpl implements EntityManager {
+ "property {} with value {}",
new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } );
- //retreive up to 100 columns
+ //retrieve up to 100 columns
List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 100,
false );
+ Entity[] entities = new Entity[cols.size()];
+ int index = 0;
- //go through it column
for ( HColumn<ByteBuffer, ByteBuffer> col : indexingColumns ) {
UUID indexCorruptionUuid = ue.fromByteBuffer( col.getName());
- if (get( indexCorruptionUuid ) == null ) {
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
- Keyspace ko = cass.getApplicationKeyspace( ownerEntityId );
- Mutator<ByteBuffer> mutator = createMutator( ko, be );
-
- addDeleteToMutator( mutator, ENTITY_UNIQUE, key, indexCorruptionUuid, timestamp );
- mutator.execute();
+
+ entities[index] = get(indexCorruptionUuid);
+
+ if (entities[index] == null ) {
+ deleteUniqueColumn( ownerEntityId, key, indexCorruptionUuid );
cols.remove( col );
}
else{
-
+ index++;
+ }
+ }
+ //this means that the same unique rowkey has two values associated with it
+ if(entities[0]!=null && entities[1]!=null){
+ Entity mostRecentEntity = entities[0];
+ for(Entity entity: entities){
+ if(mostRecentEntity.getModified() > entity.getModified()){
+ deleteEntity( entity.getUuid() );
+ logger.info( "Deleting " + entity.getUuid().toString()
+ + " because it shares older unique value with: " + propertyValue );
+ }
+ else if (mostRecentEntity.getModified() < entity.getModified()){
+ logger.info("Deleting "+mostRecentEntity.getUuid().toString()+" because it shares older unique value with: "+propertyValue);
+ deleteEntity( mostRecentEntity.getUuid() );
+ mostRecentEntity = entity;
+ }
+ else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
+ logger.info("Entities with unique value: "+propertyValue+" has two or more entities with the same modified time."
+ + "Please manually resolve by query or changing names. ");
+ }
}
}
}
@@ -603,6 +621,18 @@ public class EntityManagerImpl implements EntityManager {
}
+ private void deleteUniqueColumn( final UUID ownerEntityId, final Object key, final UUID indexCorruptionUuid )
+ throws Exception {
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+ Keyspace ko = cass.getApplicationKeyspace( ownerEntityId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, indexCorruptionUuid, timestamp );
+ mutator.execute();
+ }
+
+
/** Add this unique index to the delete */
private void uniquePropertyDelete( Mutator<ByteBuffer> m, String collectionName, String entityType,
String propertyName, Object propertyValue, UUID entityId, long timestamp )
[21/39] usergrid git commit: Added message that indicates it has
completed
Posted by sn...@apache.org.
Added message that indicates it has completed
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7db57696
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7db57696
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7db57696
Branch: refs/heads/1.x
Commit: 7db57696f70859c73cde5c32773e0339f17790bf
Parents: 0ba197e
Author: George Reyes <gr...@apache.org>
Authored: Tue Nov 10 09:05:21 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Tue Nov 10 09:05:21 2015 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7db57696/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index fe3a74e..e835bec 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -217,6 +217,7 @@ public class UniqueIndexCleanup extends ToolBase {
}
}
}
+ System.out.println("Completed repair.");
logger.info( "Completed audit of apps" );
}
[24/39] usergrid git commit: Added prints for values that end up
getting cleaned.
Posted by sn...@apache.org.
Added prints for values that end up getting cleaned.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1fab1db9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1fab1db9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1fab1db9
Branch: refs/heads/1.x
Commit: 1fab1db945da08e4bd989aa44799133428cada5f
Parents: 2e8bc46
Author: George Reyes <gr...@apache.org>
Authored: Tue Nov 10 14:15:30 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Tue Nov 10 14:15:30 2015 -0800
----------------------------------------------------------------------
.../main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1fab1db9/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index c385b50..d1a63b6 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -325,8 +325,10 @@ public class UniqueIndexCleanup extends ToolBase {
for(Entity entity: entities){
if(mostRecentEntity.getModified() > entity.getModified()){
em.deleteEntity( entity.getUuid() );
+ System.out.println("Deleting "+entity.getUuid().toString()+" because it shares older unique value with: "+uniqueValue);
}
else if (mostRecentEntity.getModified() < entity.getModified()){
+ System.out.println("Deleting "+mostRecentEntity.getUuid().toString()+" because it shares older unique value with: "+uniqueValue);
em.deleteEntity( mostRecentEntity.getUuid() );
mostRecentEntity = entity;
}
[15/39] usergrid git commit: Added ability to delete rows that
contained 0 columns.
Posted by sn...@apache.org.
Added ability to delete rows that contained 0 columns.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/dde3ea66
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/dde3ea66
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/dde3ea66
Branch: refs/heads/1.x
Commit: dde3ea668cd9608b2614002b66c1a0469fae71ae
Parents: 48a8006
Author: George Reyes <gr...@apache.org>
Authored: Thu Nov 5 11:38:24 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Nov 5 11:38:24 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 43 ++++++++++++--------
1 file changed, 27 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/dde3ea66/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index f795a20..b60981b 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -68,7 +68,7 @@ public class UniqueIndexCleanup extends ToolBase {
/**
*
*/
- private static final int PAGE_SIZE = 1;
+ private static final int PAGE_SIZE = 100;
private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
@@ -85,6 +85,8 @@ public class UniqueIndexCleanup extends ToolBase {
OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
.create( "host" );
+
+
options.addOption( hostOption );
return options;
@@ -136,30 +138,39 @@ public class UniqueIndexCleanup extends ToolBase {
EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
Boolean cleanup = false;
- //TODO: make parsed row key more human friendly. Anybody looking at it doesn't know what value means what.
- if ( parsedRowKey[1].equals( "users" ) ) {
+ if ( collectionName.equals( "users" ) ) {
ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
if ( columnSlice.getColumns().size() != 0 ) {
System.out.println( returnedRowKey );
List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+ if(cols.size()==0){
+ System.out.println("Found 0 uuid's associated with: "+uniqueValue);
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( m,ENTITY_UNIQUE,key,timestamp );
+ m.execute();
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- UUID entityId = ue.fromByteBuffer( col.getName() );
-
- if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
- if ( managementService.getAdminUserByUuid( entityId ) == null ) {
+ }
+ else {
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID entityId = ue.fromByteBuffer( col.getName() );
+
+ if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
+ if ( managementService.getAdminUserByUuid( entityId ) == null ) {
+ cleanup = true;
+ }
+ }
+ else if ( em.get( entityId ) == null ) {
cleanup = true;
}
- }
- else if ( em.get( entityId ) == null ) {
- cleanup = true;
- }
- if ( cleanup == true ) {
- DeleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue,
- entityId );
- cleanup = false;
+ if ( cleanup == true ) {
+ DeleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue,
+ entityId );
+ cleanup = false;
+ }
}
}
}
[16/39] usergrid git commit: Added ability to verify individual
entities and try to delete them. This functionality doesn't exist for columns
with nothing in them.
Posted by sn...@apache.org.
Added ability to verify individual entities and try to delete them. This functionality doesn't exist for columns with nothing in them.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/99af194b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/99af194b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/99af194b
Branch: refs/heads/1.x
Commit: 99af194bbd336bea71db42def275fe507d663e31
Parents: dde3ea6
Author: George Reyes <gr...@apache.org>
Authored: Thu Nov 5 12:27:39 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Nov 5 12:27:39 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 184 +++++++++++++------
1 file changed, 125 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/99af194b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index b60981b..541cc7b 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -29,7 +29,6 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.commons.io.Charsets;
import org.apache.thrift.TBaseHelper;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
@@ -43,7 +42,6 @@ import me.prettyprint.hector.api.beans.Row;
import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.RangeSlicesQuery;
-import sun.text.normalizer.UTF16;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
@@ -73,6 +71,14 @@ public class UniqueIndexCleanup extends ToolBase {
private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
+ private static final String APPLICATION_ARG = "app";
+
+ private static final String COLLECTION_ARG = "col";
+
+ private static final String ENTITY_UNIQUE_PROPERTY_NAME = "property";
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
@Override
@SuppressWarnings( "static-access" )
@@ -85,10 +91,31 @@ public class UniqueIndexCleanup extends ToolBase {
OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
.create( "host" );
-
+ options.addOption( hostOption );
+
+
+ Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "application id" ).create( APPLICATION_ARG );
+
+
+ options.addOption( appOption );
+
+ Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
+ .withDescription( "collection name" ).create( COLLECTION_ARG );
+
+ options.addOption( collectionOption );
+
+ Option entityUniquePropertyName =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_NAME ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Name" ).create( ENTITY_UNIQUE_PROPERTY_NAME );
+ options.addOption( entityUniquePropertyName );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
- options.addOption( hostOption );
return options;
}
@@ -110,67 +137,55 @@ public class UniqueIndexCleanup extends ToolBase {
Keyspace ko = cass.getUsergridApplicationKeyspace();
Mutator<ByteBuffer> m = createMutator( ko, be );
- RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
- HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
- //not sure if I trust the lower two settings as it might iterfere with paging or set
- // arbitrary limits and what I want to retrieve.
- //That needs to be verified.
- .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+ if ( line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ deleteInvalidValuesForUniqueProperty(m ,line );
+ }
+ else {
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
+ //That needs to be verified.
+ .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
- RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
- while ( rangeSlicesIterator.hasNext() ) {
- Row rangeSliceValue = rangeSlicesIterator.next();
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
- ByteBuffer buf = ( TBaseHelper.rightSize(( ByteBuffer ) rangeSliceValue.getKey() ) );
- //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger ones return garbage data.
- //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
- String returnedRowKey = new String(buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(), Charset.defaultCharset()).trim();
- String[] parsedRowKey = returnedRowKey.split( ":" );
- UUID applicationId = UUID.fromString(uuidGarbageParser( parsedRowKey[0]) );
- String collectionName = parsedRowKey[1];
- String uniqueValueKey = parsedRowKey[2];
- String uniqueValue = parsedRowKey[3];
+ ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
+ // ones return garbage data.
+ //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
+ Charset.defaultCharset() ).trim();
- EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
- Boolean cleanup = false;
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+ UUID applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
- if ( collectionName.equals( "users" ) ) {
- ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
- if ( columnSlice.getColumns().size() != 0 ) {
- System.out.println( returnedRowKey );
- List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
- if(cols.size()==0){
- System.out.println("Found 0 uuid's associated with: "+uniqueValue);
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( m,ENTITY_UNIQUE,key,timestamp );
- m.execute();
+ if ( collectionName.equals( "users" ) ) {
- }
- else {
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- UUID entityId = ue.fromByteBuffer( col.getName() );
-
- if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
- if ( managementService.getAdminUserByUuid( entityId ) == null ) {
- cleanup = true;
- }
- }
- else if ( em.get( entityId ) == null ) {
- cleanup = true;
- }
-
- if ( cleanup == true ) {
- DeleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue,
- entityId );
- cleanup = false;
- }
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ if ( columnSlice.getColumns().size() != 0 ) {
+ System.out.println( returnedRowKey );
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+ if ( cols.size() == 0 ) {
+ System.out.println( "Found 0 uuid's associated with: " + uniqueValue );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp );
+ m.execute();
+ }
+ else {
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols );
}
}
}
@@ -181,14 +196,65 @@ public class UniqueIndexCleanup extends ToolBase {
}
+ private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue,
+ final List<HColumn<ByteBuffer, ByteBuffer>> cols )
+ throws Exception {
+ Boolean cleanup = false;
+ EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID entityId = ue.fromByteBuffer( col.getName() );
+
+ if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
+ if ( managementService.getAdminUserByUuid( entityId ) == null ) {
+ cleanup = true;
+ }
+ }
+ else if ( em.get( entityId ) == null ) {
+ cleanup = true;
+ }
+
+ if ( cleanup == true ) {
+ deleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue,
+ entityId );
+ cleanup = false;
+ }
+ }
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty(Mutator<ByteBuffer> m,CommandLine line) throws Exception{
+ UUID applicationId = UUID.fromString( line.getOptionValue( APPLICATION_ARG ) );
+ String collectionName = line.getOptionValue( COLLECTION_ARG );
+ String uniqueValueKey = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
+ false );
+
+ if ( cols.size() == 0 ) {
+ System.out.println("Zero entities were found for this unique value. Its possible it doesn't exist.");
+ }
+
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols );
+
+ }
+
private String uuidGarbageParser( final String garbageString ) {
int index = 1;
String stringToBeTruncated = garbageString;
- while( !UUIDUtils.isUUID( stringToBeTruncated ) ){
- if( stringToBeTruncated.length()>36)
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
else {
- System.out.println(garbageString+" is unparsable");
+ System.out.println( garbageString + " is unparsable" );
break;
}
}
@@ -196,7 +262,7 @@ public class UniqueIndexCleanup extends ToolBase {
}
- private void DeleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ private void deleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
final String uniqueValueKey, final String uniqueValue, final UUID entityId )
throws Exception {
logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
[05/39] usergrid git commit: Finished refactor and verified locally.
Posted by sn...@apache.org.
Finished refactor and verified locally.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a4205230
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a4205230
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a4205230
Branch: refs/heads/1.x
Commit: a4205230cc342398e184fa87470bda93ee7fe996
Parents: 0b3eea1
Author: George Reyes <gr...@apache.org>
Authored: Mon Nov 2 09:54:43 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Mon Nov 2 09:54:43 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/usergrid/tools/CollectionUserFix.java | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4205230/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
index 9b95c7a..143685c 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
@@ -154,12 +154,7 @@ public class CollectionUserFix extends ExportingToolBase {
for ( Application application : app ) {
- if(entityManager == null || !em.getApplication().equals( application )){
- em = emf.getEntityManager( application.getUuid() );
- }
- else {
- em = entityManager;
- }
+ em = emf.getEntityManager( application.getUuid() );
PagingResultsIterator pagingResultsIterator =
new PagingResultsIterator( em.searchCollection( application, "users", query ) );
@@ -168,10 +163,9 @@ public class CollectionUserFix extends ExportingToolBase {
Entity entity = ( Entity ) pagingResultsIterator.next();
String username = entity.getProperty( "username" ).toString().toLowerCase();
em.getUserByIdentifier( identifier.fromName( username ) );
-
}
- System.out.println("Repair Complete");
+ System.out.println("Repair of application: "+ application.getApplicationName() + " complete");
}
}
}
[02/39] usergrid git commit: Added fix where when not returning all
entities would prematurely exit the program
Posted by sn...@apache.org.
Added fix where when not returning all entities would prematurely exit the program
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b44e537d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b44e537d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b44e537d
Branch: refs/heads/1.x
Commit: b44e537d5ceca642ffc3049f982e1fa41ceb374c
Parents: a72da10
Author: George Reyes <gr...@apache.org>
Authored: Mon Nov 2 08:42:05 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Mon Nov 2 08:42:05 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/CollectionUserFix.java | 131 ++++++++++---------
1 file changed, 70 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b44e537d/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
index 829487a..2e66858 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
@@ -55,7 +55,7 @@ import com.google.common.collect.Multimap;
* user shows up there. If you see that then run this tool.
*/
public class CollectionUserFix extends ExportingToolBase {
- private static final int PAGE_SIZE = 100;
+ private static final int PAGE_SIZE = 1000;
private static final Logger logger = LoggerFactory.getLogger( CollectionUserFix.class );
@@ -159,12 +159,12 @@ public class CollectionUserFix extends ExportingToolBase {
query.setLimit( PAGE_SIZE );
Results r = null;
EntityManager em = null;
+ int numberOfUsers = 0;
for ( Application application : app ) {
//This will hold all of the applications users. This will be stored in memory to do a simple check to see if
//there are any duped usernames in the collection.
//Memory concerns means that
- Multimap<String, UUID> usernames = HashMultimap.create();
//This means that we need to set it for each and every single application thus it gets set here instead of
//the method that calls us.
@@ -176,10 +176,12 @@ public class CollectionUserFix extends ExportingToolBase {
}
do {
+ Multimap<String, UUID> usernames = HashMultimap.create();
//get all users in the management app and page for each set of a PAGE_SIZE
r = em.searchCollection( application, "users", query );
- System.out.println("found "+r.size()+" number of entities");
+ numberOfUsers+=r.size();
+ System.out.println("found "+numberOfUsers+" users");
for ( Entity entity : r.getEntities() ) {
//grab all usernames returned.
@@ -188,68 +190,75 @@ public class CollectionUserFix extends ExportingToolBase {
query.setCursor( r.getCursor() );
- logger.info( "Searching next page" );
- System.out.println("Searching next page");
- }
- while ( r != null && r.size() == PAGE_SIZE );
+ System.out.println("Starting username crawl of "+usernames.size()+" number of usernames");
+ usernameVerificationFix( em, usernames );
+
- System.out.println("Starting username crawl of "+usernames.size()+" number of usernames");
+ }
+ while ( r != null && r.size() >0);
+ System.out.println("Repair Complete");
//do a get on a specific username, if it shows up more than once then remove it
- for ( String username : usernames.keySet() ) {
- Collection<UUID> ids = usernames.get( username );
+ }
+ }
- if ( ids.size() > 1 ) {
- logger.info( "Found multiple users with the username {}", username );
- System.out.println( "Found multiple users with the username: " + username );
- }
- //UserInfo targetUser = managementService.getAdminUserByEmail( email );
- Identifier identifier = new Identifier();
- EntityRef targetUser = em.getUserByIdentifier( identifier.fromName( username ) );
-
-
- if ( targetUser == null ) {
- //This means that the username isn't properly associated with targetUser
- List<UUID> tempIds = new ArrayList<UUID>( ids );
- //Collections.sort( tempIds );
-
- UUID toLoad = tempIds.get( 0 );
-
- logger.warn( "Could not load target user by username {}, loading by UUID {} instead", username, toLoad );
- System.out.println( "Could not load the target user by username: " + username
- + ". Loading by the following uuid instead: " + toLoad.toString() );
-
- User targetUserEntity = null;
- try {
- targetUserEntity = em.get( toLoad, User.class );
- }catch(Exception e){
- System.out.println("The follow uuid has no data in this cassandra node: "+toLoad.toString());
- throw e;
- }
-
-
- try {
- if ( targetUserEntity != null&& targetUserEntity.getUuid().equals( toLoad )) {
- System.out.println("Updating uuid: "+targetUserEntity.getUuid().toString());
- em.update( targetUserEntity );
- }
- }
- catch ( DuplicateUniquePropertyExistsException dup ) {
- System.out.println( "Found duplicate unique property: " + dup.getPropertyName() + ". "
- + "Duplicate property is: "
- + dup.getPropertyValue() );
- //if there are duplicate unique properties then
- if ( dup.getPropertyName().equals( "username" ) ) {
- System.out.println("can I replace this with a different value since these are duplicated in the code base");
- //targetUserEntity.setUsername( targetUserEntity.getUsername() );
- }
- //else throw dup;
- }
- catch (Exception e){
- System.out.println("There was an issue with updating: "+e.getMessage());
- }
- }
- }
+ private void usernameVerificationFix( final EntityManager em, final Multimap<String, UUID> usernames )
+ throws Exception {
+ Identifier identifier = new Identifier();
+ for ( String username : usernames.keySet() ) {
+ // Collection<UUID> ids = usernames.get( username );
+
+// if ( ids.size() > 1 ) {
+// logger.info( "Found multiple users with the username {}", username );
+// System.out.println( "Found multiple users with the username: " + username );
+// }
+
+ //UserInfo targetUser = managementService.getAdminUserByEmail( email );
+ em.getUserByIdentifier( identifier.fromName( username ) );
+
+
+
+// if ( targetUser == null ) {
+// //This means that the username isn't properly associated with targetUser
+// List<UUID> tempIds = new ArrayList<UUID>( ids );
+// //Collections.sort( tempIds );
+//
+// UUID toLoad = tempIds.get( 0 );
+//
+// logger.warn( "Could not load target user by username {}, loading by UUID {} instead", username, toLoad );
+// System.out.println( "Could not load the target user by username: " + username
+// + ". Loading by the following uuid instead: " + toLoad.toString() );
+//
+// User targetUserEntity = null;
+// try {
+// targetUserEntity = em.get( toLoad, User.class );
+// }catch(Exception e){
+// System.out.println("The follow uuid has no data in this cassandra node: "+toLoad.toString());
+// throw e;
+// }
+//
+//
+// try {
+// if ( targetUserEntity != null&& targetUserEntity.getUuid().equals( toLoad )) {
+// System.out.println("Updating uuid: "+targetUserEntity.getUuid().toString());
+// em.update( targetUserEntity );
+// }
+// }
+// catch ( DuplicateUniquePropertyExistsException dup ) {
+// System.out.println( "Found duplicate unique property: " + dup.getPropertyName() + ". "
+// + "Duplicate property is: "
+// + dup.getPropertyValue() );
+// //if there are duplicate unique properties then
+// if ( dup.getPropertyName().equals( "username" ) ) {
+// System.out.println("can I replace this with a different value since these are duplicated in the code base");
+// //targetUserEntity.setUsername( targetUserEntity.getUsername() );
+// }
+// //else throw dup;
+// }
+// catch (Exception e){
+// System.out.println("There was an issue with updating: "+e.getMessage());
+// }
+// }
}
}
}
[04/39] usergrid git commit: Removed commented out older code.
Posted by sn...@apache.org.
Removed commented out older code.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0b3eea17
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0b3eea17
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0b3eea17
Branch: refs/heads/1.x
Commit: 0b3eea1793405a089857d3c14cc898b6a83cd41a
Parents: a40cf28
Author: George Reyes <gr...@apache.org>
Authored: Mon Nov 2 09:38:32 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Mon Nov 2 09:38:32 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/CollectionUserFix.java | 127 +------------------
1 file changed, 4 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0b3eea17/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
index e63bca8..9b95c7a 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
@@ -104,15 +104,6 @@ public class CollectionUserFix extends ExportingToolBase {
/**
- * psudeo code
- * if org id is present then ignore the application id
- * go through everysingle application/users entity. And check to see if it can be queried by username.
- * else if app id is present
- * go through the applications/users
- *
- *
- * Take the list of applications to go through and go through them one by one. ( in the latter else case we will only
- *
* @param line
* @throws Exception
*/
@@ -149,8 +140,6 @@ public class CollectionUserFix extends ExportingToolBase {
System.out.println("Repair Complete");
}
-
- //make
private void startCollectionFlow(final EntityManager entityManager, final Set<Application> app, final String queryString )
throws Exception {// search for all orgs
@@ -159,51 +148,21 @@ public class CollectionUserFix extends ExportingToolBase {
query = query.fromQL( queryString );
}
query.setLimit( PAGE_SIZE );
- Results r = null;
EntityManager em = null;
- int numberOfUsers = 0;
Identifier identifier = new Identifier();
for ( Application application : app ) {
- //This will hold all of the applications users. This will be stored in memory to do a simple check to see if
- //there are any duped usernames in the collection.
- //Memory concerns means that
- //This means that we need to set it for each and every single application thus it gets set here instead of
- //the method that calls us.
- if(entityManager == null){
+ if(entityManager == null || !em.getApplication().equals( application )){
em = emf.getEntityManager( application.getUuid() );
}
else {
em = entityManager;
}
-//
-// do {
-// Multimap<String, UUID> usernames = HashMultimap.create();
-//
-//
-// //get all users in the management app and page for each set of a PAGE_SIZE
-// r = em.searchCollection( application, "users", query );
-// numberOfUsers+=r.size();
-// System.out.println("found "+numberOfUsers+" users");
-//
-// for ( Entity entity : r.getEntities() ) {
-// //grab all usernames returned.
-// usernames.put( entity.getProperty( "username" ).toString().toLowerCase(), entity.getUuid() );
-// }
-//
-// query.setCursor( r.getCursor() );
-//
-// System.out.println("Starting username crawl of "+usernames.size()+" number of usernames");
-// usernameVerificationFix( em, usernames );
-//
-//
-// }
-// while ( r != null && r.size() == PAGE_SIZE);
-
- r = em.searchCollection( application, "users", query );
- PagingResultsIterator pagingResultsIterator = new PagingResultsIterator( r );
+
+ PagingResultsIterator pagingResultsIterator =
+ new PagingResultsIterator( em.searchCollection( application, "users", query ) );
while(pagingResultsIterator.hasNext()){
Entity entity = ( Entity ) pagingResultsIterator.next();
@@ -211,86 +170,8 @@ public class CollectionUserFix extends ExportingToolBase {
em.getUserByIdentifier( identifier.fromName( username ) );
}
-//
-// for ( Entity entity : r.getEntities() ) {
-// //grab all usernames returned.
-// usernames.put( entity.getProperty( "username" ).toString().toLowerCase(), entity.getUuid() );
-// }
-//
-// query.setCursor( r.getCursor() );
-//
-// System.out.println("Starting username crawl of "+usernames.size()+" number of usernames");
-// usernameVerificationFix( em, entity.getProperty( "username" ).toString().toLowerCase());
-
-
-
-
-
-
System.out.println("Repair Complete");
- //do a get on a specific username, if it shows up more than once then remove it
- }
- }
-
-
- private void usernameVerificationFix( final EntityManager em, final Multimap<String, UUID> usernames )
- throws Exception {
- Identifier identifier = new Identifier();
- for ( String username : usernames.keySet() ) {
- // Collection<UUID> ids = usernames.get( username );
-
-// if ( ids.size() > 1 ) {
-// logger.info( "Found multiple users with the username {}", username );
-// System.out.println( "Found multiple users with the username: " + username );
-// }
-
- //UserInfo targetUser = managementService.getAdminUserByEmail( email );
- em.getUserByIdentifier( identifier.fromName( username ) );
-
-
-
-// if ( targetUser == null ) {
-// //This means that the username isn't properly associated with targetUser
-// List<UUID> tempIds = new ArrayList<UUID>( ids );
-// //Collections.sort( tempIds );
-//
-// UUID toLoad = tempIds.get( 0 );
-//
-// logger.warn( "Could not load target user by username {}, loading by UUID {} instead", username, toLoad );
-// System.out.println( "Could not load the target user by username: " + username
-// + ". Loading by the following uuid instead: " + toLoad.toString() );
-//
-// User targetUserEntity = null;
-// try {
-// targetUserEntity = em.get( toLoad, User.class );
-// }catch(Exception e){
-// System.out.println("The follow uuid has no data in this cassandra node: "+toLoad.toString());
-// throw e;
-// }
-//
-//
-// try {
-// if ( targetUserEntity != null&& targetUserEntity.getUuid().equals( toLoad )) {
-// System.out.println("Updating uuid: "+targetUserEntity.getUuid().toString());
-// em.update( targetUserEntity );
-// }
-// }
-// catch ( DuplicateUniquePropertyExistsException dup ) {
-// System.out.println( "Found duplicate unique property: " + dup.getPropertyName() + ". "
-// + "Duplicate property is: "
-// + dup.getPropertyValue() );
-// //if there are duplicate unique properties then
-// if ( dup.getPropertyName().equals( "username" ) ) {
-// System.out.println("can I replace this with a different value since these are duplicated in the code base");
-// //targetUserEntity.setUsername( targetUserEntity.getUsername() );
-// }
-// //else throw dup;
-// }
-// catch (Exception e){
-// System.out.println("There was an issue with updating: "+e.getMessage());
-// }
-// }
}
}
}
[28/39] usergrid git commit: Added in extra logic to handle legacy
entities and re added in the single case repair.
Posted by sn...@apache.org.
Added in extra logic to handle legacy entities and re added in the single case repair.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6acc8d3b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6acc8d3b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6acc8d3b
Branch: refs/heads/1.x
Commit: 6acc8d3b5eddbe5a449dee70fe910f8098cf5baf
Parents: 1fab1db
Author: George Reyes <gr...@apache.org>
Authored: Wed Nov 11 14:42:48 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Wed Nov 11 14:42:48 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 70 +++++++++++++++++---
.../usergrid/tools/UniqueIndexCleanupTest.java | 20 +++++-
2 files changed, 78 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6acc8d3b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index d1a63b6..82378cd 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -208,7 +208,7 @@ public class UniqueIndexCleanup extends ToolBase {
deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
}
else {
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols );
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,returnedRowKey );
}
}
}
@@ -275,7 +275,7 @@ public class UniqueIndexCleanup extends ToolBase {
private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
final String uniqueValueKey, final String uniqueValue,
- final List<HColumn<ByteBuffer, ByteBuffer>> cols ) throws Exception {
+ final List<HColumn<ByteBuffer, ByteBuffer>> cols, String rowKey ) throws Exception {
Boolean cleanup = false;
EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
int numberOfColumnsDeleted = 0;
@@ -320,16 +320,43 @@ public class UniqueIndexCleanup extends ToolBase {
}
//this means that the same unique rowkey has two values associated with it
- if(entities[0]!=null && entities[1]!=null){
+ if(index>2){
Entity mostRecentEntity = entities[0];
for(Entity entity: entities){
+ if(mostRecentEntity == null){
+ System.out.println( "Most Recent entity is null and is being replaced by regular entity" );
+ mostRecentEntity = entity;
+ }
+ if(entity == null){
+ System.out.println("Entity we're cycling through is null and we need more new entities");
+ continue;
+ }
+
+ mostRecentEntity = verifyModifiedTimestamp( mostRecentEntity );
+ entity = verifyModifiedTimestamp( entity );
+
+
if(mostRecentEntity.getModified() > entity.getModified()){
- em.deleteEntity( entity.getUuid() );
- System.out.println("Deleting "+entity.getUuid().toString()+" because it shares older unique value with: "+uniqueValue);
+ System.out.println("Deleting "+entity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
+ System.out.flush();
+ // try {
+ em.deleteEntity( entity.getUuid() );
+// }catch(Exception e){
+// System.out.println("Found error when trying to delete the following uuid: "+entity.getUuid()+" Please repair manually or remote debug.");
+// System.out.println(e.getMessage());
+// break;
+// }
}
else if (mostRecentEntity.getModified() < entity.getModified()){
- System.out.println("Deleting "+mostRecentEntity.getUuid().toString()+" because it shares older unique value with: "+uniqueValue);
- em.deleteEntity( mostRecentEntity.getUuid() );
+ System.out.println("Deleting "+mostRecentEntity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
+ System.out.flush();
+ //try {
+ em.deleteEntity( mostRecentEntity.getUuid() );
+// }catch(Exception e){
+// System.out.println("Found error when trying to delete the following uuid: "+mostRecentEntity.getUuid()+" Please repair manually or remote debug.");
+// System.out.println(e.getMessage());
+// break;
+// }
mostRecentEntity = entity;
}
else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
@@ -349,6 +376,26 @@ public class UniqueIndexCleanup extends ToolBase {
}
+ private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
+ Entity entity = unverifiedEntity;
+ if(entity !=null && entity.getModified()==null) {
+ if(entity.getCreated()!=null){
+ System.out.println("Setting modified timestamp to created for comparison purposes");
+ entity.setModified( entity.getCreated() );
+ return entity;
+ }
+ else{
+ System.out.println("Please delete or remake: "+entity.getUuid());
+ System.out.println("Setting timestamps to 1");
+ entity.setCreated( 1L );
+ entity.setModified( 1L );
+ return entity;
+ }
+ }
+ return entity;
+ }
+
+
//really only deletes ones that aren't existant for a specific value
private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
UUID applicationId = UUID.fromString( line.getOptionValue( APPLICATION_ARG ) );
@@ -365,12 +412,13 @@ public class UniqueIndexCleanup extends ToolBase {
if ( cols.size() == 0 ) {
- System.out.println(
- "Zero entities were found for this unique value. Its possible it doesn't exist or you typed in in"
- + " wrong :p." );
+ System.out.println("Zero columns were found for "+ key.toString()+ ". Will delete rowkey.");
+// System.out.println(
+// "Zero entities were found for this unique value. Its possible it doesn't exist or you typed in in"
+// + " wrong :p." );
}
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols );
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,key.toString() );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6acc8d3b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
index 088d7e8..e016284 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -444,7 +444,8 @@ public class UniqueIndexCleanupTest {
}
//POinting at single values is broken now but not entirely used right now anyways.
- @Ignore
+ //@Ignore
+ @Test
public void testRepairOfOnlyOneOfTwoColumnsWhilePointingAtSingleValue() throws Exception{
String rand = RandomStringUtils.randomAlphanumeric( 10 );
@@ -509,10 +510,12 @@ public class UniqueIndexCleanupTest {
// .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+ //NEED TO FAIL MORE GRACEFULLY
//run the cleanup
UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
uniqueIndexCleanup.startTool( new String[] {
"-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort(),
+ "-col",collectionName,
"-app",applicationInfo.getId().toString(),
"-property","username",
"-value",username
@@ -524,5 +527,20 @@ public class UniqueIndexCleanupTest {
}
+ @Test
+ public void errorchecker(){
+ System.out.println( "Started" );
+
+ UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+ uniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:9160",
+ "-col","users",
+ "-app","00000000-0000-0000-0000-000000000001",
+ "-property","username",
+ "-value","jromero"
+ }, false );
+ System.out.println( "Finished" );
+ }
+
}
[03/39] usergrid git commit: Changed logic to use paging results
iterator. Should be equivalent to previous iterators but easier to
understand.
Posted by sn...@apache.org.
Changed logic to use paging results iterator. Should be equivalent to previous iterators but easier to understand.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a40cf284
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a40cf284
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a40cf284
Branch: refs/heads/1.x
Commit: a40cf28430e7b480502180ea9f0a13908def4175
Parents: b44e537
Author: George Reyes <gr...@apache.org>
Authored: Mon Nov 2 09:33:36 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Mon Nov 2 09:33:36 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/CollectionUserFix.java | 62 +++++++++++++++-----
1 file changed, 47 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a40cf284/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
index 2e66858..e63bca8 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
@@ -37,6 +37,7 @@ import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.Identifier;
+import org.apache.usergrid.persistence.PagingResultsIterator;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.cassandra.CassandraService;
@@ -54,6 +55,7 @@ import com.google.common.collect.Multimap;
* the error "Could not find organization for email" because the user can't be found but when the org is queried the
* user shows up there. If you see that then run this tool.
*/
+//If index corruptions show up then the tool won't complete.
public class CollectionUserFix extends ExportingToolBase {
private static final int PAGE_SIZE = 1000;
@@ -160,6 +162,8 @@ public class CollectionUserFix extends ExportingToolBase {
Results r = null;
EntityManager em = null;
int numberOfUsers = 0;
+ Identifier identifier = new Identifier();
+
for ( Application application : app ) {
//This will hold all of the applications users. This will be stored in memory to do a simple check to see if
@@ -174,28 +178,56 @@ public class CollectionUserFix extends ExportingToolBase {
else {
em = entityManager;
}
+//
+// do {
+// Multimap<String, UUID> usernames = HashMultimap.create();
+//
+//
+// //get all users in the management app and page for each set of a PAGE_SIZE
+// r = em.searchCollection( application, "users", query );
+// numberOfUsers+=r.size();
+// System.out.println("found "+numberOfUsers+" users");
+//
+// for ( Entity entity : r.getEntities() ) {
+// //grab all usernames returned.
+// usernames.put( entity.getProperty( "username" ).toString().toLowerCase(), entity.getUuid() );
+// }
+//
+// query.setCursor( r.getCursor() );
+//
+// System.out.println("Starting username crawl of "+usernames.size()+" number of usernames");
+// usernameVerificationFix( em, usernames );
+//
+//
+// }
+// while ( r != null && r.size() == PAGE_SIZE);
+
+ r = em.searchCollection( application, "users", query );
+ PagingResultsIterator pagingResultsIterator = new PagingResultsIterator( r );
+
+ while(pagingResultsIterator.hasNext()){
+ Entity entity = ( Entity ) pagingResultsIterator.next();
+ String username = entity.getProperty( "username" ).toString().toLowerCase();
+ em.getUserByIdentifier( identifier.fromName( username ) );
+
+ }
+//
+// for ( Entity entity : r.getEntities() ) {
+// //grab all usernames returned.
+// usernames.put( entity.getProperty( "username" ).toString().toLowerCase(), entity.getUuid() );
+// }
+//
+// query.setCursor( r.getCursor() );
+//
+// System.out.println("Starting username crawl of "+usernames.size()+" number of usernames");
+// usernameVerificationFix( em, entity.getProperty( "username" ).toString().toLowerCase());
- do {
- Multimap<String, UUID> usernames = HashMultimap.create();
- //get all users in the management app and page for each set of a PAGE_SIZE
- r = em.searchCollection( application, "users", query );
- numberOfUsers+=r.size();
- System.out.println("found "+numberOfUsers+" users");
- for ( Entity entity : r.getEntities() ) {
- //grab all usernames returned.
- usernames.put( entity.getProperty( "username" ).toString().toLowerCase(), entity.getUuid() );
- }
- query.setCursor( r.getCursor() );
- System.out.println("Starting username crawl of "+usernames.size()+" number of usernames");
- usernameVerificationFix( em, usernames );
- }
- while ( r != null && r.size() >0);
System.out.println("Repair Complete");
//do a get on a specific username, if it shows up more than once then remove it
}
[37/39] usergrid git commit: Added some additional comments and final
look over of entity manager read repair.
Posted by sn...@apache.org.
Added some additional comments and final look over of entity manager read repair.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3a2a88c4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3a2a88c4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3a2a88c4
Branch: refs/heads/1.x
Commit: 3a2a88c405ccf6a7a7fa1fd9d19e8a1067a2f3f0
Parents: 15e25de
Author: George Reyes <gr...@apache.org>
Authored: Fri Nov 13 09:35:57 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Nov 13 09:35:57 2015 -0800
----------------------------------------------------------------------
.../usergrid/persistence/cassandra/EntityManagerImpl.java | 2 --
.../java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java | 6 ++----
2 files changed, 2 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3a2a88c4/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 80d9f44..38a80a0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -587,8 +587,6 @@ public class EntityManagerImpl implements EntityManager {
if (entities[index] == null ) {
deleteUniqueColumn( ownerEntityId, key, indexCorruptionUuid );
- //iffy since cols won't have the same values of indexingColumns.
- //cols.remove( col );
}
else{
index++;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3a2a88c4/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
index d95a6e7..aec4266 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
@@ -55,8 +55,6 @@ import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-//TODO: write docs so support can run it.
-//TODO: provide support with log4j file and instructions on how to use it.
/**
*
*This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users
@@ -64,11 +62,11 @@ import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
* If there exists more than one existing column then the one with the most recent timestamp wins and the other is deleted.
*
*If you want the run the tool on their cluster the following is what you need to do
- * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here> > log.txt
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host <cassandra_host_here> > log.txt
*
* if there is a specific value you want to run the tool on then you need the following
- * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here>
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UserUniqueIndexCleanup -host <cassandra_host_here>
* -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value <unique_property_value> > log.txt
*
* @author grey
[30/39] usergrid git commit: Added some fixes to read repair.
Posted by sn...@apache.org.
Added some fixes to read repair.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1d89dbb6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1d89dbb6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1d89dbb6
Branch: refs/heads/1.x
Commit: 1d89dbb69fb260f450aa5ac3f43e70c8a095ed85
Parents: b38f5bc
Author: George Reyes <gr...@apache.org>
Authored: Thu Nov 12 09:31:50 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Nov 12 09:31:50 2015 -0800
----------------------------------------------------------------------
.../apache/usergrid/persistence/cassandra/EntityManagerImpl.java | 2 +-
.../test/java/org/apache/usergrid/services/ServiceRequestIT.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d89dbb6/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index bd7773a..ea8c28e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -593,7 +593,7 @@ public class EntityManagerImpl implements EntityManager {
}
}
//this means that the same unique rowkey has two values associated with it
- if(entities[0]!=null && entities[1]!=null){
+ if(index>1){
Entity mostRecentEntity = entities[0];
for(Entity entity: entities){
if(mostRecentEntity.getModified() > entity.getModified()){
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1d89dbb6/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
index c678ba9..846c58e 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
@@ -226,7 +226,7 @@ public class ServiceRequestIT {
throw e;
}
- //verifies it works now.
+ //verifies it works when doing a get/read repair
assertNotNull( entityManager
.get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
[18/39] usergrid git commit: Added additional tests to verify that
proper values won't get deleted.
Posted by sn...@apache.org.
Added additional tests to verify that proper values won't get deleted.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/80ad870c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/80ad870c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/80ad870c
Branch: refs/heads/1.x
Commit: 80ad870caad0f3ba8e446925c61cc5b374211108
Parents: 267ea39
Author: George Reyes <gr...@apache.org>
Authored: Fri Nov 6 14:11:29 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Nov 6 14:11:29 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 35 ++-
.../usergrid/tools/UniqueIndexCleanupTest.java | 261 ++++++++++++++++++-
2 files changed, 284 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/80ad870c/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 6e20379..e1de85a 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -178,12 +178,7 @@ public class UniqueIndexCleanup extends ToolBase {
System.out.println( returnedRowKey );
List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
if ( cols.size() == 0 ) {
- System.out.println( "Found 0 uuid's associated with: " + uniqueValue );
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp );
- m.execute();
+ deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
}
else {
entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols );
@@ -197,13 +192,29 @@ public class UniqueIndexCleanup extends ToolBase {
}
+ private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue ) throws Exception {
+ System.out.println( "Found 0 uuid's associated with: " + uniqueValue );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key,timestamp );
+ mutator.execute();
+ return;
+ }
+
+
private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
final String uniqueValueKey, final String uniqueValue,
final List<HColumn<ByteBuffer, ByteBuffer>> cols )
throws Exception {
Boolean cleanup = false;
EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
-
+ int numberOfColumnsDeleted = 0;
for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
UUID entityId = ue.fromByteBuffer( col.getName() );
@@ -217,11 +228,15 @@ public class UniqueIndexCleanup extends ToolBase {
}
if ( cleanup == true ) {
+ numberOfColumnsDeleted++;
deleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue,
entityId );
cleanup = false;
}
}
+ if(cols.size()==numberOfColumnsDeleted){
+ deleteRow( m,applicationId,collectionName,uniqueValueKey,uniqueValue );
+ }
}
@@ -270,10 +285,12 @@ public class UniqueIndexCleanup extends ToolBase {
System.out.println( "Deleting column uuid: " + entityId.toString() );
UUID timestampUuid = newTimeUUID();
long timestamp = getTimestampInMicros( timestampUuid );
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
- m.execute();
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, entityId, timestamp );
+ mutator.execute();
return;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/80ad870c/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
index ddf7e88..b3b7896 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -45,6 +45,9 @@ import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.utils.UUIDUtils;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.HColumn;
@@ -54,13 +57,17 @@ import me.prettyprint.hector.api.mutation.Mutator;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
/**
@@ -105,8 +112,9 @@ public class UniqueIndexCleanupTest {
System.out.println( "completed" );
}
+ //this test is perfect for the other tool the userCollectionFix tool as this is what I believe they were seeing.
@Test
- public void testRepairOfSingleEntity() throws Exception{
+ public void testRepairOfSingleEntityMissingColumnWrongTool() throws Exception{
String rand = RandomStringUtils.randomAlphanumeric( 10 );
String orgName = "org_" + rand;
@@ -154,15 +162,262 @@ public class UniqueIndexCleanupTest {
key = key( applicationInfo.getId(), collectionName, "username", username );
//addDeleteToMutator( m, ENTITY_UNIQUE, key, uuid, timestamp );
addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, uuid );
+ m.execute();
+ assertNull( entityManager.getAlias( applicationInfo.getId(), collectionName, username ) );
- MutationResult mutationResult= m.execute();
+ assertNotNull(entityManager.get( entityToBeCorrupted.getUuid() ));
+ //run the cleanup
+ UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+ uniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
- assertNull( entityManager.getAlias( applicationInfo.getId(),collectionName,username));
//here you need to add a delete to the mutator then recheck it and see if the entity is the same as millicoms.
+ Thread.sleep( 2000 );
+ assertNull( entityManager.get( entityToBeCorrupted.getUuid() ) );
+
+ //When you come back you also need to emulate the tests to delete what is out of the uuid without any corresponding data.
+ //maybe it'll be easier to just do an insert into the EntityUnique row without doint it into any other row and
+ //then verify the data like that. Then you don't have to do deletes out of other things.
+
+ }
+
+ //For this test you need to insert a dummy key with a dummy column that leads to nowhere
+ //then run the unique index cleanup.
+ //checks for bug when only column doesn't exist make sure to delete the row as well.
+ @Test
+ public void testRepairOfSingleEntity() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+
+ //run the cleanup
+ UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+ uniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
+
+
+ entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ assertNotNull( entityToBeCorrupted );
+ assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername());
+
+ }
+
+ //For this test you need to insert a dummy key with a dummy column that leads to nowhere
+ //then run the unique index cleanup.
+ //checks for bug when only column doesn't exist make sure to delete the row as well.
+ @Test
+ public void testRepairOfMultipleEntities() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ int numberOfEntitiesToCreate = 1000;
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ String[] usernames = new String[numberOfEntitiesToCreate];
+
+ int index = 0;
+ while(index < numberOfEntitiesToCreate) {
+
+ usernames[index]=username+index;
+
+ Map<String, Object> userInfo = new HashMap<String, Object>();
+ userInfo.put( "username", usernames[index] );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", usernames[index] );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name
+ addInsertToMutator( m, ENTITY_UNIQUE, key, testEntityUUID, 0, createTimestamp() );
+
+ m.execute();
+ index++;
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+ }
+
+
+
+ //run the cleanup
+ UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+ uniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
+
+ for(String user:usernames ) {
+ Map<String, Object> userInfo = new HashMap<String, Object>();
+ userInfo.put( "username", user);
+ Entity entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+
+ assertNotNull( entityToBeCorrupted );
+ assertEquals( user, ( ( User ) entityToBeCorrupted ).getUsername() );
+ }
+ }
+
+ @Test
+ public void testRepairOfOnlyOneOfTwoColumns() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ //create a new column
+ Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+ //should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
+ assertNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+
+ //run the cleanup
+ UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+ uniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+ }, false );
+ //verifies it works now.
+ assertNotNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
}
[09/39] usergrid git commit: Created initial version that actually
iterates through entity unique column and finds uuid's to corresponding
entities.
Posted by sn...@apache.org.
Created initial version that actually iterates through entity unique column and finds uuid's to corresponding entities.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/bd1acf57
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/bd1acf57
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/bd1acf57
Branch: refs/heads/1.x
Commit: bd1acf578faa7747380dcb1f3d5b2dd564d74008
Parents: ac7e30a
Author: George Reyes <gr...@apache.org>
Authored: Wed Nov 4 15:41:46 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Wed Nov 4 15:41:46 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 249 +++++++++----------
1 file changed, 122 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/bd1acf57/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index f86da7b..e03e286 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -30,10 +30,12 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.db.RowIteratorFactory;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.commons.io.Charsets;
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.persistence.Entity;
@@ -43,30 +45,51 @@ import org.apache.usergrid.persistence.Identifier;
import org.apache.usergrid.persistence.IndexBucketLocator;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.persistence.cassandra.Serializers;
+import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
+import org.apache.usergrid.persistence.cassandra.index.UUIDStartToBytes;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser;
import org.apache.usergrid.persistence.schema.CollectionInfo;
+import me.prettyprint.cassandra.service.KeyIterator;
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.AbstractComposite.ComponentEquality;
+import me.prettyprint.hector.api.beans.ColumnSlice;
import me.prettyprint.hector.api.beans.DynamicComposite;
import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.OrderedRows;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.beans.Rows;
+import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.QueryResult;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+import me.prettyprint.hector.api.query.SliceQuery;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static me.prettyprint.hector.api.factory.HFactory.createRangeSlicesQuery;
+import static me.prettyprint.hector.api.factory.HFactory.createSliceQuery;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_COLLECTIONS;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_ID_SETS;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX_ENTRIES;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_PROPERTIES;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.INDEX_ENTRY_LIST_COUNT;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.APPLICATIONS_CF;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.utils.CompositeUtils.setEqualityFlag;
+import static org.apache.usergrid.persistence.cassandra.Serializers.dce;
+import static org.apache.usergrid.persistence.cassandra.Serializers.le;
+import static org.apache.usergrid.persistence.cassandra.Serializers.se;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.ConversionUtils.bytebuffer;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
@@ -127,7 +150,7 @@ public class UniqueIndexCleanup extends ToolBase {
options.addOption( appOption );
Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
- .withDescription( "colleciton name" ).create( COLLECTION_ARG );
+ .withDescription( "collection name" ).create( COLLECTION_ARG );
options.addOption( collectionOption );
@@ -166,9 +189,10 @@ public class UniqueIndexCleanup extends ToolBase {
}
CassandraService cass = em.getCass();
- IndexBucketLocator indexBucketLocator = em.getIndexBucketLocator();
- Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
UUID timestampUuid = newTimeUUID();
long timestamp = getTimestampInMicros( timestampUuid );
@@ -177,159 +201,130 @@ public class UniqueIndexCleanup extends ToolBase {
// go through each collection and audit the values
for ( String collectionName : getCollectionNames( em, line ) ) {
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery = HFactory
+ .createRangeSlicesQuery( ko, be, be, be )
+ .setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two ssettings as it might iterfere with paging or set arbitrary limits and what I want to retrieve.
+ //That needs to be verified.
+ .setKeys( null, null )
+ .setRange( null, null, false, 100 );
- for ( final String bucketName : indexBucketLocator.getBuckets() ) {
-
- IndexScanner scanner =
- cass.getIdList( key( applicationId, ENTITY_UNIQUE, collectionName ), null, null,
- PAGE_SIZE, false, bucketName, applicationId, false );
- SliceIterator itr = new SliceIterator( scanner, new UUIDIndexSliceParser( null ) );
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery,null,null );
+ QueryResult<OrderedRows<ByteBuffer, ByteBuffer, ByteBuffer>> result = rangeSlicesQuery.execute();
+ OrderedRows<ByteBuffer, ByteBuffer, ByteBuffer> rows = result.get();
+ result.get().getList().get( 0 ).getColumnSlice();
- while ( itr.hasNext() ) {
+ while(rangeSlicesIterator.hasNext()){
+ //UUID returned_uuid = UUID.nameUUIDFromBytes(((ByteBuffer)rangeSlicesIterator.next().getKey()).array());
+ Row rangeSliceValue = rangeSlicesIterator.next();
- Set<ScanColumn> ids = itr.next();
+ String returnedRowKey = new String( ((ByteBuffer)rangeSliceValue.getKey()).array(), Charsets.UTF_8);
+ if ( returnedRowKey.contains( "users" ) || returnedRowKey.contains( "username" ) || returnedRowKey.contains( "email" ) ) {
+ ColumnSlice<ByteBuffer,ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ if(columnSlice.getColumns().size()!=0) {
+ System.out.println( returnedRowKey );
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
- CollectionInfo collection = getDefaultSchema().getCollection( "application", collectionName );
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID entityId = ue.fromByteBuffer( col.getName() );
+ if(em.get( entityId )==null){
+ logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
+ System.out.println("Deleting column uuid: "+entityId.toString());
- //We shouldn't have to do this, but otherwise the cursor won't work
- Set<String> indexed = collection.getPropertiesIndexed();
- // what's left needs deleted, do so
+ Object key = key( applicationId, collectionName,"username", entityId );
+ addDeleteToMutator(m,ENTITY_UNIQUE,key,entityId,timestamp);
+ m.execute();
+ continue;
+ }
+ }
- logger.info( "Auditing {} entities for collection {} in app {}", new Object[] {
- ids.size(), collectionName, app.getValue()
- } );
- for ( ScanColumn col : ids ) {
- final UUID id = col.getUUID();
- boolean reIndex = false;
+ // }
+ }
+ // rangeSlicesIterator.next()
- Mutator<ByteBuffer> m = createMutator( ko, be );
+ // UUID.nameUUIDFromBytes(rows.getList().get( 0 ).getKey().array());
- try {
- Entity entity = em.get( id );
- //entity may not exist, but we should have deleted rows from the index
- if ( entity == null ) {
- logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
- //now execute the cleanup. In this case the entity is gone,
- // so we'll want to remove references from
- // the secondary index also remove from unique entity.
- Object key = key( applicationId, collectionName,"username", id );
- addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
- m.execute();
- continue;
- }
- logger.info( "Reindex complete for entity with id '{} ", id );
- //now execute the cleanup. This way if the above update fails,
- // we still have enough data to run again
- // later
- m.execute();
+ // for ( String key : keyIterator ) {
+// System.out.printf("Current key: %s \n",key);
+// keyCount++;
+// }
+// System.out.printf( "Iterated over %d keys \n", keyCount );
//
-// for ( String prop : indexed ) {
-//
-// String bucket = indexBucketLocator
-// .getBucket( id );
+// for ( final String bucketName : indexBucketLocator.getBuckets() ) {
//
-// Object rowKey = key( applicationId, collection.getName(), prop, bucket );
+// System.out.println("Entity Unique to be scanned");
+// Set<String> columns = cass.getAllColumnNames(cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE,key( applicationId, collectionName,"username" ) );
+// Rows<UUID, String, ByteBuffer> results =
+// cass.getRows( cass.getApplicationKeyspace( applicationId ), ENTITY_PROPERTIES, entityIds, ue, se, be );
+
+
+ //Set results = cass.getRowKeySet( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, Serializers.ue );
+ //Set<String> columnStrings = cass.getAllColumnNames(cass.getApplicationKeyspace( applicationId ),ENTITY_UNIQUE, key( applicationId, collectionName ) );
+// Object key = createUniqueIndexKey( ownerEntityId, collectionNameInternal, propertyName, propertyValue );
//
-// List<HColumn<ByteBuffer, ByteBuffer>> indexCols =
-// scanIndexForAllTypes( ko, indexBucketLocator, applicationId, rowKey, id,
-// prop );
+// List<HColumn<ByteBuffer, ByteBuffer>> cols =
+// cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
+// false );
+
+
+// IndexScanner scanner =new IndexBucketScanner<UUID>( cass, ENTITY_UNIQUE, UUIDStartToBytes.INSTANCE, applicationId, key( applicationId, collectionName,"username" )
+// , bucketName, null,
+// null, false, PAGE_SIZE, true );
+//// cass.getIdList( key( applicationId, DICTIONARY_, collectionName ), null, null,
+//// PAGE_SIZE, false, bucketName, applicationId, false );
//
-// // loop through the indexed values and verify them as present in
-// // our entity_index_entries. If they aren't, we need to delete the
-// // from the secondary index, and mark
-// // this object for re-index via n update
-// for ( HColumn<ByteBuffer, ByteBuffer> index : indexCols ) {
//
-// DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( index
-// .getName().duplicate() );
+// SliceIterator itr = new SliceIterator( scanner, new UUIDIndexSliceParser( null ) );
//
-// Object code = secondaryIndexValue.get( 0 );
-// Object propValue = secondaryIndexValue.get( 1 );
-// UUID timestampId = ( UUID ) secondaryIndexValue.get( 3 );
//
-// DynamicComposite existingEntryStart = new DynamicComposite( prop, code,
-// propValue, timestampId );
-// DynamicComposite existingEntryFinish = new DynamicComposite( prop, code,
-// propValue, timestampId );
+// while ( itr.hasNext() ) {
+// System.out.println("Iterating on collections.");
//
-// setEqualityFlag( existingEntryFinish, ComponentEquality.GREATER_THAN_EQUAL );
+// Set<ScanColumn> ids = itr.next();
//
-// // now search our EntityIndexEntry for previous values, see if
-// // they don't match this one
+// CollectionInfo collection = getDefaultSchema().getCollection( "application", collectionName );
//
-// List<HColumn<ByteBuffer, ByteBuffer>> entries =
-// cass.getColumns( ko, ENTITY_INDEX_ENTRIES, id, existingEntryStart,
-// existingEntryFinish, INDEX_ENTRY_LIST_COUNT, false );
+// logger.info( "Auditing {} entities for collection {} in app {}", new Object[] {
+// ids.size(), collectionName, app.getValue()
+// } );
//
-// // we wouldn't find this column in our entity_index_entries
-// // audit. Delete it, then mark this entity for update
-// if ( entries.size() == 0 ) {
-// logger.info(
-// "Could not find reference to value '{}' for property '{}' on entity "
-// +
-// "{} in collection {}. " + " Forcing reindex", new Object[] { propValue, prop, id, collectionName } );
+// for ( ScanColumn col : ids ) {
+// final UUID id = col.getUUID();
+// Mutator<ByteBuffer> m = createMutator( ko, be );
//
-// addDeleteToMutator( m, ENTITY_INDEX, rowKey, index.getName().duplicate(),
-// timestamp );
+// try {
+// System.out.println("Verifying uuid: "+id.toString());
+// Entity entity = em.get( id );
//
-// reIndex = true;
-// }
+// //entity may not exist, but we should have deleted rows from the index
+// if ( entity == null ) {
+// logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
+// System.out.println("Deleting uuid: "+id.toString());
//
-// if ( entries.size() > 1 ) {
-// Object key = key( applicationId, collectionName, prop, id );
-// addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
-// logger.info(
-// "Found more than 1 entity referencing unique index for property "
-// + "'{}' "
-// +
-// "with value " + "'{}'", prop, propValue );
-// reIndex = true;
-// }
-// }
+// Object key = key( applicationId, collectionName,"username", id );
+// addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
+// m.execute();
+// continue;
// }
-
- //force this entity to be updated
- if ( reIndex ) {
- Entity entity = em.get( id );
-
- //entity may not exist, but we should have deleted rows from the index
- if ( entity == null ) {
- logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
- //now execute the cleanup. In this case the entity is gone,
- // so we'll want to remove references from
- // the secondary index also remove from unique entity.
-
-
- m.execute();
- continue;
- }
-
-
- logger.info( "Reindex complete for entity with id '{} ", id );
- em.update( entity );
-
- //now execute the cleanup. This way if the above update fails,
- // we still have enough data to run again
- // later
- m.execute();
- }
- }
- catch ( Exception e ) {
- logger.error( "Unable to process entity with id '{}'", id, e );
- }
- }
- }
- }
+// logger.info( "Reindex complete for entity with id '{} ", id );
+// m.execute();
+// }
+// catch ( Exception e ) {
+// logger.error( "Unable to process entity with id '{}'", id, e );
+// }
+// }
+// }
+// }
}
}
[10/39] usergrid git commit: Cleaned up code that was left over from
dev process.
Posted by sn...@apache.org.
Cleaned up code that was left over from dev process.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/94826997
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/94826997
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/94826997
Branch: refs/heads/1.x
Commit: 94826997754300b034754860424e5987a9f2ab7e
Parents: bd1acf5
Author: George Reyes <gr...@apache.org>
Authored: Wed Nov 4 16:01:57 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Wed Nov 4 16:01:57 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 121 ++++---------------
1 file changed, 23 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/94826997/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index e03e286..ba7441b 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -216,115 +216,40 @@ public class UniqueIndexCleanup extends ToolBase {
OrderedRows<ByteBuffer, ByteBuffer, ByteBuffer> rows = result.get();
result.get().getList().get( 0 ).getColumnSlice();
- while(rangeSlicesIterator.hasNext()){
+ while(rangeSlicesIterator.hasNext()) {
//UUID returned_uuid = UUID.nameUUIDFromBytes(((ByteBuffer)rangeSlicesIterator.next().getKey()).array());
Row rangeSliceValue = rangeSlicesIterator.next();
- String returnedRowKey = new String( ((ByteBuffer)rangeSliceValue.getKey()).array(), Charsets.UTF_8);
- if ( returnedRowKey.contains( "users" ) || returnedRowKey.contains( "username" ) || returnedRowKey.contains( "email" ) ) {
- ColumnSlice<ByteBuffer,ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
- if(columnSlice.getColumns().size()!=0) {
+ String returnedRowKey =
+ new String( ( ( ByteBuffer ) rangeSliceValue.getKey() ).array(), Charsets.UTF_8 ).trim();
+
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+ if ( parsedRowKey[1].equals( "users" ) || returnedRowKey.contains( "username" ) || returnedRowKey
+ .contains( "email" ) ) {
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ if ( columnSlice.getColumns().size() != 0 ) {
System.out.println( returnedRowKey );
- List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID entityId = ue.fromByteBuffer( col.getName() );
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- UUID entityId = ue.fromByteBuffer( col.getName() );
- if(em.get( entityId )==null){
- logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
- System.out.println("Deleting column uuid: "+entityId.toString());
+ if ( em.get( entityId ) == null && managementService.getAdminUserByUuid( entityId )==null ) {
+ logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
+ System.out.println( "Deleting column uuid: " + entityId.toString() );
- Object key = key( applicationId, collectionName,"username", entityId );
- addDeleteToMutator(m,ENTITY_UNIQUE,key,entityId,timestamp);
- m.execute();
- continue;
+ Object key = key( applicationId, collectionName, "username", entityId );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
+ m.execute();
+ continue;
+ }
}
}
-
-
- // }
+ }
}
- // rangeSlicesIterator.next()
-
- // UUID.nameUUIDFromBytes(rows.getList().get( 0 ).getKey().array());
-
-
-
-
-
- // for ( String key : keyIterator ) {
-// System.out.printf("Current key: %s \n",key);
-// keyCount++;
-// }
-// System.out.printf( "Iterated over %d keys \n", keyCount );
-
-//
-// for ( final String bucketName : indexBucketLocator.getBuckets() ) {
-//
-// System.out.println("Entity Unique to be scanned");
-// Set<String> columns = cass.getAllColumnNames(cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE,key( applicationId, collectionName,"username" ) );
-// Rows<UUID, String, ByteBuffer> results =
-// cass.getRows( cass.getApplicationKeyspace( applicationId ), ENTITY_PROPERTIES, entityIds, ue, se, be );
-
-
- //Set results = cass.getRowKeySet( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, Serializers.ue );
- //Set<String> columnStrings = cass.getAllColumnNames(cass.getApplicationKeyspace( applicationId ),ENTITY_UNIQUE, key( applicationId, collectionName ) );
-// Object key = createUniqueIndexKey( ownerEntityId, collectionNameInternal, propertyName, propertyValue );
-//
-// List<HColumn<ByteBuffer, ByteBuffer>> cols =
-// cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
-// false );
-
-
-// IndexScanner scanner =new IndexBucketScanner<UUID>( cass, ENTITY_UNIQUE, UUIDStartToBytes.INSTANCE, applicationId, key( applicationId, collectionName,"username" )
-// , bucketName, null,
-// null, false, PAGE_SIZE, true );
-//// cass.getIdList( key( applicationId, DICTIONARY_, collectionName ), null, null,
-//// PAGE_SIZE, false, bucketName, applicationId, false );
-//
-//
-// SliceIterator itr = new SliceIterator( scanner, new UUIDIndexSliceParser( null ) );
-//
-//
-// while ( itr.hasNext() ) {
-// System.out.println("Iterating on collections.");
-//
-// Set<ScanColumn> ids = itr.next();
-//
-// CollectionInfo collection = getDefaultSchema().getCollection( "application", collectionName );
-//
-// logger.info( "Auditing {} entities for collection {} in app {}", new Object[] {
-// ids.size(), collectionName, app.getValue()
-// } );
-//
-// for ( ScanColumn col : ids ) {
-// final UUID id = col.getUUID();
-// Mutator<ByteBuffer> m = createMutator( ko, be );
-//
-// try {
-// System.out.println("Verifying uuid: "+id.toString());
-// Entity entity = em.get( id );
-//
-// //entity may not exist, but we should have deleted rows from the index
-// if ( entity == null ) {
-// logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
-// System.out.println("Deleting uuid: "+id.toString());
-//
-// Object key = key( applicationId, collectionName,"username", id );
-// addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
-// m.execute();
-// continue;
-// }
-// logger.info( "Reindex complete for entity with id '{} ", id );
-// m.execute();
-// }
-// catch ( Exception e ) {
-// logger.error( "Unable to process entity with id '{}'", id, e );
-// }
-// }
-// }
-// }
+
}
}
[29/39] usergrid git commit: Removed some comments.
Posted by sn...@apache.org.
Removed some comments.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/80ffb354
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/80ffb354
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/80ffb354
Branch: refs/heads/1.x
Commit: 80ffb35481740cde38b17d82cec699ac8992d009
Parents: 6acc8d3
Author: George Reyes <gr...@apache.org>
Authored: Wed Nov 11 14:50:12 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Wed Nov 11 14:50:12 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 25 ++++----------------
1 file changed, 5 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/80ffb354/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 82378cd..7c1fadd 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -339,24 +339,13 @@ public class UniqueIndexCleanup extends ToolBase {
if(mostRecentEntity.getModified() > entity.getModified()){
System.out.println("Deleting "+entity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
System.out.flush();
- // try {
- em.deleteEntity( entity.getUuid() );
-// }catch(Exception e){
-// System.out.println("Found error when trying to delete the following uuid: "+entity.getUuid()+" Please repair manually or remote debug.");
-// System.out.println(e.getMessage());
-// break;
-// }
+ em.deleteEntity( entity.getUuid() );
+
}
else if (mostRecentEntity.getModified() < entity.getModified()){
System.out.println("Deleting "+mostRecentEntity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
System.out.flush();
- //try {
- em.deleteEntity( mostRecentEntity.getUuid() );
-// }catch(Exception e){
-// System.out.println("Found error when trying to delete the following uuid: "+mostRecentEntity.getUuid()+" Please repair manually or remote debug.");
-// System.out.println(e.getMessage());
-// break;
-// }
+ em.deleteEntity( mostRecentEntity.getUuid() );
mostRecentEntity = entity;
}
else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
@@ -366,8 +355,7 @@ public class UniqueIndexCleanup extends ToolBase {
}
}
-
-
+
//a safer way to do this would be to try to do another get and verify there is nothing left in the column
//instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
if ( cols.size() == numberOfColumnsDeleted ) {
@@ -412,10 +400,7 @@ public class UniqueIndexCleanup extends ToolBase {
if ( cols.size() == 0 ) {
- System.out.println("Zero columns were found for "+ key.toString()+ ". Will delete rowkey.");
-// System.out.println(
-// "Zero entities were found for this unique value. Its possible it doesn't exist or you typed in in"
-// + " wrong :p." );
+ System.out.println( "Zero columns were found for " + key.toString() + ". Will delete rowkey." );
}
entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,key.toString() );
[14/39] usergrid git commit: Added a workaround for buffers starting
with garbage data instead of the application uuid. Now it filters out the
beginning of the buffer.
Posted by sn...@apache.org.
Added a workaround for buffers starting with garbage data instead of the application uuid. Now it filters out the beginning of the buffer.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/48a8006d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/48a8006d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/48a8006d
Branch: refs/heads/1.x
Commit: 48a8006d41c23abbdae23b20c851e76f7f53d3f2
Parents: 6d96a4c
Author: George Reyes <gr...@apache.org>
Authored: Thu Nov 5 11:04:57 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Nov 5 11:04:57 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 41 +++++++++++++++-----
1 file changed, 31 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/48a8006d/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 39ac762..f795a20 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.tools;
import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
import java.util.List;
import java.util.UUID;
@@ -29,8 +30,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.io.Charsets;
+import org.apache.thrift.TBaseHelper;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.utils.UUIDUtils;
import me.prettyprint.cassandra.service.RangeSlicesIterator;
import me.prettyprint.hector.api.Keyspace;
@@ -40,6 +43,7 @@ import me.prettyprint.hector.api.beans.Row;
import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.RangeSlicesQuery;
+import sun.text.normalizer.UTF16;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
@@ -64,7 +68,7 @@ public class UniqueIndexCleanup extends ToolBase {
/**
*
*/
- private static final int PAGE_SIZE = 100;
+ private static final int PAGE_SIZE = 1;
private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
@@ -100,7 +104,6 @@ public class UniqueIndexCleanup extends ToolBase {
logger.info( "Starting entity unique index cleanup" );
-
// go through each collection and audit the values
Keyspace ko = cass.getUsergridApplicationKeyspace();
Mutator<ByteBuffer> m = createMutator( ko, be );
@@ -118,11 +121,14 @@ public class UniqueIndexCleanup extends ToolBase {
while ( rangeSlicesIterator.hasNext() ) {
Row rangeSliceValue = rangeSlicesIterator.next();
- String returnedRowKey =
- new String( ( ( ByteBuffer ) rangeSliceValue.getKey() ).array(), Charsets.UTF_8 ).trim();
+
+ ByteBuffer buf = ( TBaseHelper.rightSize(( ByteBuffer ) rangeSliceValue.getKey() ) );
+ //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger ones return garbage data.
+ //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ String returnedRowKey = new String(buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(), Charset.defaultCharset()).trim();
String[] parsedRowKey = returnedRowKey.split( ":" );
- UUID applicationId = UUID.fromString( parsedRowKey[0] );
+ UUID applicationId = UUID.fromString(uuidGarbageParser( parsedRowKey[0]) );
String collectionName = parsedRowKey[1];
String uniqueValueKey = parsedRowKey[2];
String uniqueValue = parsedRowKey[3];
@@ -147,10 +153,10 @@ public class UniqueIndexCleanup extends ToolBase {
}
}
else if ( em.get( entityId ) == null ) {
- cleanup =true;
+ cleanup = true;
}
- if(cleanup == true){
+ if ( cleanup == true ) {
DeleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue,
entityId );
cleanup = false;
@@ -164,9 +170,24 @@ public class UniqueIndexCleanup extends ToolBase {
}
- private void DeleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId,
- final String collectionName, final String uniqueValueKey, final String uniqueValue,
- final UUID entityId ) throws Exception {
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while( !UUIDUtils.isUUID( stringToBeTruncated ) ){
+ if( stringToBeTruncated.length()>36)
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ else {
+ System.out.println(garbageString+" is unparsable");
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private void DeleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue, final UUID entityId )
+ throws Exception {
logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
System.out.println( "Deleting column uuid: " + entityId.toString() );
UUID timestampUuid = newTimeUUID();
[19/39] usergrid git commit: Added additional test to verify that we
can delete and fix single values
Posted by sn...@apache.org.
Added additional test to verify that we can delete and fix single values
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/14eec4f2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/14eec4f2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/14eec4f2
Branch: refs/heads/1.x
Commit: 14eec4f229c988704753083a543bf2dd378d615d
Parents: 80ad870
Author: George Reyes <gr...@apache.org>
Authored: Fri Nov 6 14:22:03 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Nov 6 14:22:03 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanupTest.java | 88 ++++++++++++++++++--
1 file changed, 81 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/14eec4f2/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
index b3b7896..315cc16 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -405,6 +405,8 @@ public class UniqueIndexCleanupTest {
}
//should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
+ //TODO: fix the below so that it fails everytime.
+ //50/50 chance to succeed or fail
assertNull( entityManager
.get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
@@ -421,13 +423,85 @@ public class UniqueIndexCleanupTest {
}
- private static int getFileCount( File exportDir, final String ext ) {
- return exportDir.listFiles( new FileFilter() {
- @Override
- public boolean accept( File pathname ) {
- return pathname.getAbsolutePath().endsWith( "." + ext );
- }
- } ).length;
+ @Test
+ public void testRepairOfOnlyOneOfTwoColumnsWhilePointingAtSingleValue() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ //create a new column
+ Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+ //should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
+ assertNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+
+ //run the cleanup
+ UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+ uniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort(),
+ "-app",applicationInfo.getId().toString(),
+ "-property","username",
+ "-value",username
+ }, false );
+
+ //verifies it works now.
+ assertNotNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
}
+
}
[39/39] usergrid git commit: this closes #437
Posted by sn...@apache.org.
this closes #437
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9f3299ef
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9f3299ef
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9f3299ef
Branch: refs/heads/1.x
Commit: 9f3299ef669fdd4920d2eb396602f3d7cca89e9e
Parents: bd4fd26 d2bd3e8
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Nov 17 17:04:08 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Nov 17 17:04:08 2015 -0500
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 80 +++
.../usergrid/persistence/EntityManagerIT.java | 2 +
.../rest/applications/users/UsersResource.java | 3 +
...cateUniquePropertyExistsExceptionMapper.java | 6 +
.../usergrid/services/ServiceRequestIT.java | 240 ++++++++-
.../usergrid/tools/CollectionUserFix.java | 171 +++++++
.../usergrid/tools/UniqueIndexCleanup.java | 387 ---------------
.../usergrid/tools/UserUniqueIndexCleanup.java | 476 ++++++++++++++++++
.../tools/UserUniqueIndexCleanupTest.java | 490 +++++++++++++++++++
9 files changed, 1467 insertions(+), 388 deletions(-)
----------------------------------------------------------------------
[23/39] usergrid git commit: Added ignore to test for use case that
doesn't currently work. Added ability to filter out multiple entities with
the same user name ( pick the one one modified last )
Posted by sn...@apache.org.
Added ignore to test for use case that doesn't currently work.
Added ability to filter out multiple entities with the same user name ( pick the one one modified last )
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2e8bc463
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2e8bc463
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2e8bc463
Branch: refs/heads/1.x
Commit: 2e8bc46329b11d5e625d75aad1b3dbb1d39e91ee
Parents: 7db5769
Author: George Reyes <gr...@apache.org>
Authored: Tue Nov 10 12:22:41 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Tue Nov 10 12:22:41 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 104 +++++++++++++------
.../usergrid/tools/UniqueIndexCleanupTest.java | 2 +-
2 files changed, 73 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e8bc463/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index e835bec..c385b50 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -31,6 +31,7 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.thrift.TBaseHelper;
+import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
import org.apache.usergrid.utils.StringUtils;
import org.apache.usergrid.utils.UUIDUtils;
@@ -138,9 +139,9 @@ public class UniqueIndexCleanup extends ToolBase {
Keyspace ko = cass.getUsergridApplicationKeyspace();
Mutator<ByteBuffer> m = createMutator( ko, be );
- if (line.hasOption( APPLICATION_ARG ) || line.hasOption( COLLECTION_ARG ) ||
- line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
- deleteInvalidValuesForUniqueProperty(m ,line );
+ if ( line.hasOption( APPLICATION_ARG ) || line.hasOption( COLLECTION_ARG ) ||
+ line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ deleteInvalidValuesForUniqueProperty( m, line );
}
else {
@@ -166,25 +167,22 @@ public class UniqueIndexCleanup extends ToolBase {
Charset.defaultCharset() ).trim();
-
-
-
//defensive programming, don't have to have to parse the string if it doesn't contain users.
- if(returnedRowKey.contains( "users" )) {
+ if ( returnedRowKey.contains( "users" ) ) {
String[] parsedRowKey = returnedRowKey.split( ":" );
//if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
if ( parsedRowKey.length > 4 ) {
- parsedRowKey = garbageRowKeyParser(parsedRowKey);
+ parsedRowKey = garbageRowKeyParser( parsedRowKey );
- if(parsedRowKey == null) {
+ if ( parsedRowKey == null ) {
System.out.println( returnedRowKey + " is a invalid row key, and unparseable. Skipped..." );
continue;
}
}
//if the rowkey contains less than four parts then it is completely invalid
- else if ( parsedRowKey.length < 4){
+ else if ( parsedRowKey.length < 4 ) {
System.out.println( returnedRowKey + " is a invalid row key and will be skipped" );
continue;
}
@@ -217,16 +215,17 @@ public class UniqueIndexCleanup extends ToolBase {
}
}
}
- System.out.println("Completed repair.");
+ System.out.println( "Completed repair." );
logger.info( "Completed audit of apps" );
}
+
//Returns a functioning rowkey if it can otherwise returns null
- public String[] garbageRowKeyParser(String[] parsedRowKey){
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
String[] modifiedRowKey = parsedRowKey.clone();
- while(modifiedRowKey!=null) {
- if(modifiedRowKey.length < 4){
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 4 ) {
return null;
}
@@ -242,15 +241,14 @@ public class UniqueIndexCleanup extends ToolBase {
}
}
return modifiedRowKey;
-
}
private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
- for( int i = 1; i < modifiedRowKey.length; i++){
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
- recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
- if(i+1 != modifiedRowKey.length){
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
recreatedRowKey = recreatedRowKey.concat( ":" );
}
}
@@ -269,7 +267,7 @@ public class UniqueIndexCleanup extends ToolBase {
Mutator<ByteBuffer> mutator = createMutator( ko, be );
Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( mutator, ENTITY_UNIQUE, key,timestamp );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, timestamp );
mutator.execute();
return;
}
@@ -277,40 +275,80 @@ public class UniqueIndexCleanup extends ToolBase {
private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
final String uniqueValueKey, final String uniqueValue,
- final List<HColumn<ByteBuffer, ByteBuffer>> cols )
- throws Exception {
+ final List<HColumn<ByteBuffer, ByteBuffer>> cols ) throws Exception {
Boolean cleanup = false;
EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
int numberOfColumnsDeleted = 0;
+ //these columns all come from the same row key, which means they each belong to the same row key identifier
+ //thus mixing and matching them in the below if cases won't matter.
+ Entity[] entities = new Entity[cols.size()];
+
+ if(cols.size() < 2){
+ entities = new Entity[2];
+ }
+
+ int index = 0;
+
for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
UUID entityId = ue.fromByteBuffer( col.getName() );
if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
- if ( managementService.getAdminUserByUuid( entityId ) == null ) {
+
+ entities[index] = managementService.getAdminUserEntityByUuid( entityId );
+ if ( entities[index] == null ) {
cleanup = true;
}
+ else {
+ index++;
+ }
}
- else if ( em.get( entityId ) == null ) {
- cleanup = true;
+ else {
+ entities[index] = em.get( entityId );
+ if ( entities[index] == null ) {
+ cleanup = true;
+ }
+ else {
+ index++;
+ }
}
if ( cleanup == true ) {
numberOfColumnsDeleted++;
- deleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue,
- entityId );
+ deleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue, entityId );
cleanup = false;
}
}
+
+ //this means that the same unique rowkey has two values associated with it
+ if(entities[0]!=null && entities[1]!=null){
+ Entity mostRecentEntity = entities[0];
+ for(Entity entity: entities){
+ if(mostRecentEntity.getModified() > entity.getModified()){
+ em.deleteEntity( entity.getUuid() );
+ }
+ else if (mostRecentEntity.getModified() < entity.getModified()){
+ em.deleteEntity( mostRecentEntity.getUuid() );
+ mostRecentEntity = entity;
+ }
+ else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
+ System.out.println("Entities with unique value: "+uniqueValue+" has two or more entities with the same modified time."
+ + "Please manually resolve by query or changing names. ");
+ }
+ }
+ }
+
+
+
//a safer way to do this would be to try to do another get and verify there is nothing left in the column
//instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
- if(cols.size()==numberOfColumnsDeleted){
- deleteRow( m,applicationId,collectionName,uniqueValueKey,uniqueValue );
+ if ( cols.size() == numberOfColumnsDeleted ) {
+ deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
}
}
//really only deletes ones that aren't existant for a specific value
- private void deleteInvalidValuesForUniqueProperty(Mutator<ByteBuffer> m,CommandLine line) throws Exception{
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
UUID applicationId = UUID.fromString( line.getOptionValue( APPLICATION_ARG ) );
String collectionName = line.getOptionValue( COLLECTION_ARG );
String uniqueValueKey = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
@@ -324,15 +362,16 @@ public class UniqueIndexCleanup extends ToolBase {
false );
-
if ( cols.size() == 0 ) {
- System.out.println("Zero entities were found for this unique value. Its possible it doesn't exist or you typed in in wrong :p.");
+ System.out.println(
+ "Zero entities were found for this unique value. Its possible it doesn't exist or you typed in in"
+ + " wrong :p." );
}
entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols );
-
}
+
private String uuidGarbageParser( final String garbageString ) {
int index = 1;
String stringToBeTruncated = garbageString;
@@ -348,6 +387,7 @@ public class UniqueIndexCleanup extends ToolBase {
return stringToBeTruncated;
}
+
private String uuidStringVerifier( final String garbageString ) {
int index = 1;
String stringToBeTruncated = garbageString;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e8bc463/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
index 664ca7c..088d7e8 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -444,7 +444,7 @@ public class UniqueIndexCleanupTest {
}
//POinting at single values is broken now but not entirely used right now anyways.
- @Test
+ @Ignore
public void testRepairOfOnlyOneOfTwoColumnsWhilePointingAtSingleValue() throws Exception{
String rand = RandomStringUtils.randomAlphanumeric( 10 );
[12/39] usergrid git commit: Removing code that has been refactored
out. Edited flow to lead up to range query with more readability.
Posted by sn...@apache.org.
Removing code that has been refactored out. Edited flow to lead up to range query with more readability.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6e8b2b62
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6e8b2b62
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6e8b2b62
Branch: refs/heads/1.x
Commit: 6e8b2b621cc45636bb322b03d0670d5d01b0a97d
Parents: 03a123d
Author: George Reyes <gr...@apache.org>
Authored: Wed Nov 4 16:34:48 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Wed Nov 4 16:34:48 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 206 +++++--------------
1 file changed, 54 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6e8b2b62/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 52288e4..9b1c202 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -98,12 +98,7 @@ import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
/**
* This is a utility to audit all available entity ids in the secondary index. It then checks to see if any index value
* is not present in the Entity_Index_Entries. If it is not, the value from the index is removed, and a forced re-index
- * is triggered
- * <p/>
- * USERGRID-323
- * <p/>
- * <p/>
- * UniqueIndexCleanup -app [appid] -col [collectionname]
+ * is triggered <p/> USERGRID-323 <p/> <p/> UniqueIndexCleanup -app [appid] -col [collectionname]
*
* @author tnine
*/
@@ -115,7 +110,6 @@ public class UniqueIndexCleanup extends ToolBase {
private static final int PAGE_SIZE = 100;
-
private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
/**
@@ -130,7 +124,7 @@ public class UniqueIndexCleanup extends ToolBase {
@Override
- @SuppressWarnings("static-access")
+ @SuppressWarnings( "static-access" )
public Options createOptions() {
@@ -169,97 +163,66 @@ public class UniqueIndexCleanup extends ToolBase {
public void runTool( CommandLine line ) throws Exception {
startSpring();
- logger.info( "Starting entity cleanup" );
-
- Map<String, UUID> apps = getApplications( emf, line );
-
-
- for ( Entry<String, UUID> app : apps.entrySet() ) {
-
- logger.info( "Starting cleanup for app {}", app.getKey() );
-
- UUID applicationId = app.getValue();
- EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
-
- //sanity check for corrupt apps
- Application appEntity = em.getApplication();
-
- if ( appEntity == null ) {
- logger.warn( "Application does not exist in data. {}", app.getKey() );
- continue;
- }
-
- CassandraService cass = em.getCass();
+ logger.info( "Starting entity unique index cleanup" );
- Keyspace ko = cass.getUsergridApplicationKeyspace();
- Mutator<ByteBuffer> m = createMutator( ko, be );
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
-
- // go through each collection and audit the values
- for ( String collectionName : getCollectionNames( em, line ) ) {
-
- RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery = HFactory
- .createRangeSlicesQuery( ko, be, be, be )
- .setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
- //not sure if I trust the lower two settings as it might iterfere with paging or set arbitrary limits and what I want to retrieve.
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
//That needs to be verified.
- .setKeys( null, null )
- .setRange( null, null, false, 100 );
-
+ .setKeys( null, null ).setRange( null, null, false, 100 );
- RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery,null,null );
- QueryResult<OrderedRows<ByteBuffer, ByteBuffer, ByteBuffer>> result = rangeSlicesQuery.execute();
- OrderedRows<ByteBuffer, ByteBuffer, ByteBuffer> rows = result.get();
- result.get().getList().get( 0 ).getColumnSlice();
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
- while(rangeSlicesIterator.hasNext()) {
- Row rangeSliceValue = rangeSlicesIterator.next();
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
- String returnedRowKey =
- new String( ( ( ByteBuffer ) rangeSliceValue.getKey() ).array(), Charsets.UTF_8 ).trim();
+ String returnedRowKey =
+ new String( ( ( ByteBuffer ) rangeSliceValue.getKey() ).array(), Charsets.UTF_8 ).trim();
- String[] parsedRowKey = returnedRowKey.split( ":" );
- if ( parsedRowKey[1].equals( "users" ) ) {
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+ UUID applicationId = UUID.fromString( parsedRowKey[0] );
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
- ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
- if ( columnSlice.getColumns().size() != 0 ) {
- System.out.println( returnedRowKey );
- List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- UUID entityId = ue.fromByteBuffer( col.getName() );
+ EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
+ Boolean cleanup = false;
- if(parsedRowKey[0].equals( MANAGEMENT_APPLICATION_ID.toString() )){
- if(managementService.getAdminUserByUuid( entityId )==null ){
- logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
- System.out.println( "Deleting column uuid: " + entityId.toString() );
+ //TODO: make parsed row key more human friendly. Anybody looking at it doesn't know what value means what.
+ if ( parsedRowKey[1].equals( "users" ) ) {
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ if ( columnSlice.getColumns().size() != 0 ) {
+ System.out.println( returnedRowKey );
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
- Object key = key( applicationId, collectionName, parsedRowKey[2], parsedRowKey[3]);
- addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
- m.execute();
- continue;
- }
- }
- else if ( em.get( entityId ) == null ) {
- logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
- System.out.println( "Deleting column uuid: " + entityId.toString() );
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID entityId = ue.fromByteBuffer( col.getName() );
- Object key = key( applicationId, collectionName, parsedRowKey[2], parsedRowKey[3]);
- addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
- m.execute();
- continue;
- }
+ if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
+ if ( managementService.getAdminUserByUuid( entityId ) == null ) {
+ cleanup = true;
}
}
+ else if ( em.get( entityId ) == null ) {
+ cleanup =true;
+ }
+
+ if(cleanup == true){
+ DeleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue,
+ entityId );
+ cleanup = false;
+ }
}
}
-
}
}
@@ -267,78 +230,17 @@ public class UniqueIndexCleanup extends ToolBase {
}
- private Map<String, UUID> getApplications( EntityManagerFactory emf, CommandLine line ) throws Exception {
- String appName = line.getOptionValue( APPLICATION_ARG );
-
- if ( appName == null ) {
- return emf.getApplications();
- }
-
- ApplicationInfo app = managementService.getApplicationInfo( Identifier.from( appName ) );
-
- if ( app == null ) {
- logger.error( "Could not find application with id or name {}", appName );
- System.exit( 3 );
- }
-
-
- Map<String, UUID> apps = new HashMap<String, UUID>();
-
- apps.put( app.getName(), app.getId() );
-
- return apps;
- }
-
-
- private Set<String> getCollectionNames( EntityManager em, CommandLine line ) throws Exception {
-
- String collectionName = line.getOptionValue( COLLECTION_ARG );
-
- if ( collectionName == null ) {
- return em.getApplicationCollections();
- }
-
-
- Set<String> names = new HashSet<String>();
- names.add( collectionName );
-
- return names;
- }
-
-
- private List<HColumn<ByteBuffer, ByteBuffer>> scanIndexForAllTypes( Keyspace ko,
- IndexBucketLocator indexBucketLocator,
- UUID applicationId, Object rowKey,
- UUID entityId, String prop ) throws Exception {
-
- //TODO Determine the index bucket. Scan the entire index for properties with this entityId.
-
-
- DynamicComposite start = null;
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols;
-
- List<HColumn<ByteBuffer, ByteBuffer>> results = new ArrayList<HColumn<ByteBuffer, ByteBuffer>>();
-
-
- do {
- cols = cass.getColumns( ko, ENTITY_INDEX, rowKey, start, null, 100, false );
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( col.getName().duplicate() );
-
- UUID storedId = ( UUID ) secondaryIndexValue.get( 2 );
-
- //add it to the set. We can't short circuit due to property ordering
- if ( entityId.equals( storedId ) ) {
- results.add( col );
- }
-
- start = secondaryIndexValue;
- }
- }
- while ( cols.size() == 100 );
+ private void DeleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId,
+ final String collectionName, final String uniqueValueKey, final String uniqueValue,
+ final UUID entityId ) throws Exception {
+ logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
+ System.out.println( "Deleting column uuid: " + entityId.toString() );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
- return results;
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
+ m.execute();
+ return;
}
}
[32/39] usergrid git commit: Cleanup tests for unique index cleanup
Posted by sn...@apache.org.
Cleanup tests for unique index cleanup
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b141565e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b141565e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b141565e
Branch: refs/heads/1.x
Commit: b141565e3740b3ef52e7ca115479f2f4f3297cf1
Parents: a12fddc
Author: George Reyes <gr...@apache.org>
Authored: Thu Nov 12 09:35:10 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Nov 12 09:35:10 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanupTest.java | 60 +-------------------
1 file changed, 2 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b141565e/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
index e016284..25dd888 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -86,24 +86,6 @@ public class UniqueIndexCleanupTest {
@org.junit.Test
public void testBasicOperation() throws Exception {
-
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- // create app with some data
-
- String orgName = "org_" + rand;
- String appName = "app_" + rand;
- //
- // ExportDataCreator creator = new ExportDataCreator();
- // creator.startTool( new String[] {
- // "-organization", orgName,
- // "-application", appName,
- // "-host", "localhost:9160" //+ ServiceITSuite.cassandraResource.getRpcPort()
- // }, false);
-
- long start = System.currentTimeMillis();
-
-
UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
uniqueIndexCleanup.startTool( new String[] {
"-host", "localhost:9160"
@@ -230,9 +212,6 @@ public class UniqueIndexCleanupTest {
//verify it returns null.
assertNull(entityManager.get( testEntityUUID ));
- //the below works but not needed for this test.
- //assertNull( entityManager.getAlias("user",username));
-
//verify that we cannot recreate the entity due to duplicate unique property exception
Entity entityToBeCorrupted = null;
try {
@@ -326,8 +305,6 @@ public class UniqueIndexCleanupTest {
}
-
-
//run the cleanup
UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
uniqueIndexCleanup.startTool( new String[] {
@@ -389,9 +366,6 @@ public class UniqueIndexCleanupTest {
//verify it returns null.
assertNull(entityManager.get( testEntityUUID ));
- //the below works but not needed for this test.
- //assertNull( entityManager.getAlias("user",username));
-
//verify that we cannot recreate the entity due to duplicate unique property exception
Entity entityToBeCorrupted = null;
try {
@@ -404,12 +378,6 @@ public class UniqueIndexCleanupTest {
fail("shouldn't throw something else i think");
}
- //should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
- //TODO: fix the below so that it fails everytime.
- //50/50 chance to succeed or fail
-// assertNull( entityManager
-// .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
-
//run the cleanup
UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
@@ -490,9 +458,6 @@ public class UniqueIndexCleanupTest {
//verify it returns null.
assertNull(entityManager.get( testEntityUUID ));
- //the below works but not needed for this test.
- //assertNull( entityManager.getAlias("user",username));
-
//verify that we cannot recreate the entity due to duplicate unique property exception
Entity entityToBeCorrupted = null;
try {
@@ -501,15 +466,10 @@ public class UniqueIndexCleanupTest {
}catch(DuplicateUniquePropertyExistsException dup){
}
- catch(Exception e){
- fail("shouldn't throw something else i think");
+ catch(Exception e) {
+ fail( "shouldn't throw something else i think" );
}
- //should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
-// assertNull( entityManager
-// .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
-
-
//NEED TO FAIL MORE GRACEFULLY
//run the cleanup
UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
@@ -526,21 +486,5 @@ public class UniqueIndexCleanupTest {
.get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
}
-
- @Test
- public void errorchecker(){
- System.out.println( "Started" );
-
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:9160",
- "-col","users",
- "-app","00000000-0000-0000-0000-000000000001",
- "-property","username",
- "-value","jromero"
- }, false );
- System.out.println( "Finished" );
- }
-
}
[31/39] usergrid git commit: Merge branch 'USERGRID-1076' of
https://git-wip-us.apache.org/repos/asf/usergrid into
readRepairForIndexValues
Posted by sn...@apache.org.
Merge branch 'USERGRID-1076' of https://git-wip-us.apache.org/repos/asf/usergrid into readRepairForIndexValues
* 'USERGRID-1076' of https://git-wip-us.apache.org/repos/asf/usergrid:
Removed some comments.
Added in extra logic to handle legacy entities and re added in the single case repair.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a12fddc0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a12fddc0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a12fddc0
Branch: refs/heads/1.x
Commit: a12fddc0475a8f3abdb60b5f247b5a06ae7e84ed
Parents: 1d89dbb 80ffb35
Author: George Reyes <gr...@apache.org>
Authored: Thu Nov 12 09:32:07 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Nov 12 09:32:07 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 55 ++++++++++++++++----
.../usergrid/tools/UniqueIndexCleanupTest.java | 20 ++++++-
2 files changed, 63 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
[38/39] usergrid git commit: Added fix to basic operation test
Posted by sn...@apache.org.
Added fix to basic operation test
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d2bd3e83
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d2bd3e83
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d2bd3e83
Branch: refs/heads/1.x
Commit: d2bd3e83ac980bee6270b24e328988d8f48b6926
Parents: 3a2a88c
Author: George Reyes <gr...@apache.org>
Authored: Fri Nov 13 10:05:50 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Nov 13 10:05:50 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2bd3e83/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
index e08b579..f537857 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UserUniqueIndexCleanupTest.java
@@ -83,7 +83,7 @@ public class UserUniqueIndexCleanupTest {
public void testBasicOperation() throws Exception {
UserUniqueIndexCleanup userUniqueIndexCleanup = new UserUniqueIndexCleanup();
userUniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:9160"
+ "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
}, false );
System.out.println( "completed" );
[06/39] usergrid git commit: Added entity unique to unique index
cleanup.
Posted by sn...@apache.org.
Added entity unique to unique index cleanup.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a4a7884b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a4a7884b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a4a7884b
Branch: refs/heads/1.x
Commit: a4a7884b8c9f1b4f996dc3b51124c72eee3e9afa
Parents: a420523
Author: George Reyes <gr...@apache.org>
Authored: Mon Nov 2 14:57:06 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Mon Nov 2 14:57:06 2015 -0800
----------------------------------------------------------------------
.../main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a4a7884b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 994e850..bf8f0f2 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -61,6 +61,7 @@ import static org.apache.usergrid.persistence.Schema.DICTIONARY_COLLECTIONS;
import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX_ENTRIES;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraService.INDEX_ENTRY_LIST_COUNT;
@@ -255,6 +256,8 @@ public class UniqueIndexCleanup extends ToolBase {
"Could not find reference to value '{}' for property '{}' on entity "
+
"{} in collection {}. " + " Forcing reindex", new Object[] { propValue, prop, id, collectionName } );
+ Object key = key( applicationId, collectionName, prop, id );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
addDeleteToMutator( m, ENTITY_INDEX, rowKey, index.getName().duplicate(),
timestamp );
@@ -263,6 +266,8 @@ public class UniqueIndexCleanup extends ToolBase {
}
if ( entries.size() > 1 ) {
+ Object key = key( applicationId, collectionName, prop, id );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
logger.info(
"Found more than 1 entity referencing unique index for property "
+ "'{}' "
[33/39] usergrid git commit: Added retries for timeouts in unique
index cleanup
Posted by sn...@apache.org.
Added retries for timeouts in unique index cleanup
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/91b5ab23
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/91b5ab23
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/91b5ab23
Branch: refs/heads/1.x
Commit: 91b5ab2323837443c6628f644c52a1e67f2bb074
Parents: b141565
Author: George Reyes <gr...@apache.org>
Authored: Thu Nov 12 11:12:34 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Nov 12 11:12:34 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 21 ++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/91b5ab23/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 7c1fadd..64b634c 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -25,6 +25,7 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.thrift.TimedOutException;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
@@ -282,6 +283,7 @@ public class UniqueIndexCleanup extends ToolBase {
//these columns all come from the same row key, which means they each belong to the same row key identifier
//thus mixing and matching them in the below if cases won't matter.
Entity[] entities = new Entity[cols.size()];
+ int numberOfRetrys = 5;
if(cols.size() < 2){
entities = new Entity[2];
@@ -293,8 +295,15 @@ public class UniqueIndexCleanup extends ToolBase {
UUID entityId = ue.fromByteBuffer( col.getName() );
if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
+ for(int i = 0; i<numberOfRetrys; i++){
+ try {
+ entities[index] = managementService.getAdminUserEntityByUuid( entityId );
+ break;
+ }catch(TimedOutException toe){
+ Thread.sleep( 2000 );
+ }
+ }
- entities[index] = managementService.getAdminUserEntityByUuid( entityId );
if ( entities[index] == null ) {
cleanup = true;
}
@@ -303,7 +312,15 @@ public class UniqueIndexCleanup extends ToolBase {
}
}
else {
- entities[index] = em.get( entityId );
+
+ for(int i = 0; i<numberOfRetrys; i++){
+ try {
+ entities[index] = em.get( entityId );
+ break;
+ }catch(TimedOutException toe){
+ Thread.sleep( 2000 );
+ }
+ }
if ( entities[index] == null ) {
cleanup = true;
}
[34/39] usergrid git commit: Ignored two tests that are long longer
valid tools test since the inclusion of a read repair. They are now part of
the service tests.
Posted by sn...@apache.org.
Ignored two tests that are long longer valid tools test since the inclusion of a read repair. They are now part of the service tests.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a874aa90
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a874aa90
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a874aa90
Branch: refs/heads/1.x
Commit: a874aa901d6c869e580430fe5e947316aa33649a
Parents: 91b5ab2
Author: George Reyes <gr...@apache.org>
Authored: Thu Nov 12 12:04:50 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Thu Nov 12 12:04:50 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanupTest.java | 21 ++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a874aa90/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
index 25dd888..0d921d7 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -170,7 +170,9 @@ public class UniqueIndexCleanupTest {
//For this test you need to insert a dummy key with a dummy column that leads to nowhere
//then run the unique index cleanup.
//checks for bug when only column doesn't exist make sure to delete the row as well.
- @Test
+
+ //due to the read repair this is no longer a valid test of hte unique index cleanup
+ @Ignore
public void testRepairOfSingleEntity() throws Exception{
String rand = RandomStringUtils.randomAlphanumeric( 10 );
@@ -210,15 +212,16 @@ public class UniqueIndexCleanupTest {
//verify that there is no corresponding entity with the uuid or alias provided
//verify it returns null.
- assertNull(entityManager.get( testEntityUUID ));
+ //assertNull(entityManager.get( testEntityUUID ));
//verify that we cannot recreate the entity due to duplicate unique property exception
+ //The Get above should have repaired the entity allowing it run
Entity entityToBeCorrupted = null;
try {
entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
+ //fail();
}catch(DuplicateUniquePropertyExistsException dup){
-
+ fail();
}
catch(Exception e){
fail("shouldn't throw something else i think");
@@ -226,10 +229,10 @@ public class UniqueIndexCleanupTest {
//run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
+// UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+// uniqueIndexCleanup.startTool( new String[] {
+// "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
+// }, false );
entityToBeCorrupted = entityManager.create( collectionName,userInfo );
@@ -242,7 +245,9 @@ public class UniqueIndexCleanupTest {
//For this test you need to insert a dummy key with a dummy column that leads to nowhere
//then run the unique index cleanup.
//checks for bug when only column doesn't exist make sure to delete the row as well.
+ // Due to the read repair this is no longer a valid test of unique index cleanup.
@Test
+ @Ignore
public void testRepairOfMultipleEntities() throws Exception{
String rand = RandomStringUtils.randomAlphanumeric( 10 );
[27/39] usergrid git commit: Added cleanup for false unique property
cleanup if there were no corresponding entities Added test for 1000 such
entities and verifying that they were indeed fixed correctly.
Posted by sn...@apache.org.
Added cleanup for false unique property cleanup if there were no corresponding entities
Added test for 1000 such entities and verifying that they were indeed fixed correctly.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b38f5bcb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b38f5bcb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b38f5bcb
Branch: refs/heads/1.x
Commit: b38f5bcb48dd2603a0b095d2c651ba2575a006f1
Parents: 3d5ae46
Author: George Reyes <gr...@apache.org>
Authored: Tue Nov 10 16:00:08 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Tue Nov 10 16:00:08 2015 -0800
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 12 +++
.../usergrid/services/ServiceRequestIT.java | 107 +++++++++++++++----
2 files changed, 101 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b38f5bcb/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 76cb6d7..bd7773a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -547,6 +547,18 @@ public class EntityManagerImpl implements EntityManager {
false );
+ //check to see if the single value is valid. If it is not valid then it will be zero and go through
+ //the code below.
+ if(cols.size() == 1){
+ logger.info("Verifying that column is still valid, if not will be removed");
+ UUID indexCorruptionUuid = ue.fromByteBuffer( cols.get( 0 ).getName());
+
+ if (get(indexCorruptionUuid) == null ) {
+ deleteUniqueColumn( ownerEntityId, key, indexCorruptionUuid );
+ cols.remove( 0 );
+ }
+ }
+
//No columns at all, it's unique
if ( cols.size() == 0 ) {
return Collections.emptySet();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b38f5bcb/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
index d89536f..c678ba9 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
@@ -31,6 +31,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.lang.RandomStringUtils;
+import org.apache.http.annotation.NotThreadSafe;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
@@ -41,6 +42,8 @@ import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.management.OrganizationOwnerInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.entities.User;
@@ -64,7 +67,7 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.fail;
-@Concurrent()
+@NotThreadSafe()
public class ServiceRequestIT {
private static final Logger logger = LoggerFactory.getLogger( ServiceRequestIT.class );
@@ -153,16 +156,9 @@ public class ServiceRequestIT {
Entity entityToBeCorrupted = null;
try {
entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
+ }catch(Exception e){
+ fail("Create call should have worked");
}
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
-
- entityToBeCorrupted = entityManager.create( collectionName,userInfo );
assertNotNull( entityToBeCorrupted );
assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername() );
@@ -230,17 +226,92 @@ public class ServiceRequestIT {
throw e;
}
- //should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
- //TODO: fix the below so that it fails everytime.
- //50/50 chance to succeed or fail
- // assertNull( entityManager
- // .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
-
-
-
//verifies it works now.
assertNotNull( entityManager
.get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
}
+
+ //For this test you need to insert a dummy key with a dummy column that leads to nowhere
+ //then run the unique index cleanup.
+ //checks for bug when only column doesn't exist make sure to delete the row as well.
+ @Test
+ public void testRepairOfMultipleEntitiesAndRemainingEntities() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ int numberOfEntitiesToCreate = 1000;
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ String[] usernames = new String[numberOfEntitiesToCreate];
+ Entity[] entities = new Entity[numberOfEntitiesToCreate];
+
+ int index = 0;
+ while(index < numberOfEntitiesToCreate) {
+
+ usernames[index]=username+index;
+
+ Map<String, Object> userInfo = new HashMap<String, Object>();
+ userInfo.put( "username", usernames[index] );
+
+ CassandraService cass = setup.getCassSvc();
+
+ entities[index] = entityManager.create( collectionName,userInfo );
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", usernames[index] );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name
+ addInsertToMutator( m, ENTITY_UNIQUE, key, testEntityUUID, 0, createTimestamp() );
+
+ m.execute();
+ index++;
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+ }
+
+
+ for(String user:usernames ) {
+ Map<String, Object> userInfo = new HashMap<String, Object>();
+ userInfo.put( "username", user);
+
+
+ EntityRef entityRef = entityManager.getAlias( collectionName,user );
+
+ Entity entityToBeCorrupted = entityManager.get( entityRef );
+ assertNotNull( entityToBeCorrupted );
+ assertEquals( user, ( ( User ) entityToBeCorrupted ).getUsername() );
+ }
+ }
}
[17/39] usergrid git commit: Created test for exact issue. Will try
repair on that
Posted by sn...@apache.org.
Created test for exact issue. Will try repair on that
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/267ea394
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/267ea394
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/267ea394
Branch: refs/heads/1.x
Commit: 267ea394021f027306e7821e8ef3f938f2bd44a0
Parents: 99af194
Author: George Reyes <gr...@apache.org>
Authored: Fri Nov 6 07:37:11 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Nov 6 07:37:11 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 5 +-
.../usergrid/tools/UniqueIndexCleanupTest.java | 166 ++++++++++++++-----
2 files changed, 132 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/267ea394/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 541cc7b..6e20379 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -137,7 +137,8 @@ public class UniqueIndexCleanup extends ToolBase {
Keyspace ko = cass.getUsergridApplicationKeyspace();
Mutator<ByteBuffer> m = createMutator( ko, be );
- if ( line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ if (line.hasOption( APPLICATION_ARG ) || line.hasOption( COLLECTION_ARG ) ||
+ line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
deleteInvalidValuesForUniqueProperty(m ,line );
}
else {
@@ -239,7 +240,7 @@ public class UniqueIndexCleanup extends ToolBase {
false );
if ( cols.size() == 0 ) {
- System.out.println("Zero entities were found for this unique value. Its possible it doesn't exist.");
+ System.out.println("Zero entities were found for this unique value. Its possible it doesn't exist or you typed in in wrong :p.");
}
entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/267ea394/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
index 194793c..ddf7e88 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -19,8 +19,17 @@ package org.apache.usergrid.tools;
import java.io.File;
import java.io.FileFilter;
-
-import org.junit.ClassRule;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.junit.*;
+import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,58 +38,141 @@ import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.ServiceITSuite;
-
-import static org.junit.Assert.assertTrue;
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.management.OrganizationOwnerInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.mutation.MutationResult;
+import me.prettyprint.hector.api.mutation.Mutator;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
/**
* Created by ApigeeCorporation on 11/2/15.
*/
public class UniqueIndexCleanupTest {
- static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
+ static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
- int NUM_COLLECTIONS = 10;
- int NUM_ENTITIES = 50;
- int NUM_CONNECTIONS = 3;
+ int NUM_COLLECTIONS = 10;
+ int NUM_ENTITIES = 50;
+ int NUM_CONNECTIONS = 3;
- @ClassRule
- public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
+ @ClassRule
+ public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
- @org.junit.Test
- public void testBasicOperation() throws Exception {
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
+ @org.junit.Test
+ public void testBasicOperation() throws Exception {
- // create app with some data
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
- String orgName = "org_" + rand;
- String appName = "app_" + rand;
-//
-// ExportDataCreator creator = new ExportDataCreator();
-// creator.startTool( new String[] {
-// "-organization", orgName,
-// "-application", appName,
-// "-host", "localhost:9160" //+ ServiceITSuite.cassandraResource.getRpcPort()
-// }, false);
+ // create app with some data
- long start = System.currentTimeMillis();
+ String orgName = "org_" + rand;
+ String appName = "app_" + rand;
+ //
+ // ExportDataCreator creator = new ExportDataCreator();
+ // creator.startTool( new String[] {
+ // "-organization", orgName,
+ // "-application", appName,
+ // "-host", "localhost:9160" //+ ServiceITSuite.cassandraResource.getRpcPort()
+ // }, false);
+ long start = System.currentTimeMillis();
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[]{
- "-host", "localhost:9160"
- }, false );
- System.out.println("completed");
- }
+ UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+ uniqueIndexCleanup.startTool( new String[] {
+ "-host", "localhost:9160"
+ }, false );
+
+ System.out.println( "completed" );
+ }
+
+ @Test
+ public void testRepairOfSingleEntity() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,username,username,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+ CassandraService cass = setup.getCassSvc();
- private static int getFileCount(File exportDir, final String ext ) {
- return exportDir.listFiles( new FileFilter() {
- @Override
- public boolean accept(File pathname) {
- return pathname.getAbsolutePath().endsWith("." + ext);
- }
- } ).length;
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationInfo.getId() ), ENTITY_UNIQUE, key, null, null,
+ 2, false );
+
+ Set<UUID> results = new HashSet<UUID>( cols.size() );
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ results.add( ue.fromByteBuffer( col.getName() ) );
}
+
+ UUID uuid = results.iterator().next();
+
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+
+ //Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ key = key( applicationInfo.getId(), collectionName, "username", username );
+ //addDeleteToMutator( m, ENTITY_UNIQUE, key, uuid, timestamp );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, uuid );
+
+
+ MutationResult mutationResult= m.execute();
+
+
+ assertNull( entityManager.getAlias( applicationInfo.getId(),collectionName,username));
+
+ //here you need to add a delete to the mutator then recheck it and see if the entity is the same as millicoms.
+
+
+ }
+
+ private static int getFileCount( File exportDir, final String ext ) {
+ return exportDir.listFiles( new FileFilter() {
+ @Override
+ public boolean accept( File pathname ) {
+ return pathname.getAbsolutePath().endsWith( "." + ext );
+ }
+ } ).length;
+ }
}
[08/39] usergrid git commit: Committing smoke test to step through
the new unique index cleanup
Posted by sn...@apache.org.
Committing smoke test to step through the new unique index cleanup
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ac7e30a8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ac7e30a8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ac7e30a8
Branch: refs/heads/1.x
Commit: ac7e30a871ebb3701bc3d62f3b5923f9e0e3e332
Parents: 5dabe6e
Author: George Reyes <gr...@apache.org>
Authored: Tue Nov 3 15:56:44 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Tue Nov 3 15:56:44 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanupTest.java | 88 ++++++++++++++++++++
1 file changed, 88 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ac7e30a8/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
new file mode 100644
index 0000000..8cb6a88
--- /dev/null
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.io.File;
+import java.io.FileFilter;
+
+import org.junit.ClassRule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.RandomStringUtils;
+
+import org.apache.usergrid.ServiceITSetup;
+import org.apache.usergrid.ServiceITSetupImpl;
+import org.apache.usergrid.ServiceITSuite;
+
+import static org.junit.Assert.assertTrue;
+
+
+/**
+ * Created by ApigeeCorporation on 11/2/15.
+ */
+public class UniqueIndexCleanupTest {
+ static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
+
+ int NUM_COLLECTIONS = 10;
+ int NUM_ENTITIES = 50;
+ int NUM_CONNECTIONS = 3;
+
+ @ClassRule
+ public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
+
+ @org.junit.Test
+ public void testBasicOperation() throws Exception {
+
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ // create app with some data
+
+ String orgName = "org_" + rand;
+ String appName = "app_" + rand;
+//
+// ExportDataCreator creator = new ExportDataCreator();
+// creator.startTool( new String[] {
+// "-organization", orgName,
+// "-application", appName,
+// "-host", "localhost:9160" //+ ServiceITSuite.cassandraResource.getRpcPort()
+// }, false);
+
+ long start = System.currentTimeMillis();
+
+
+ UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
+ uniqueIndexCleanup.startTool( new String[]{
+ "-app", "942712f0-7ce2-11e5-b81a-17ac5477fa5c",
+ "-col", "users",
+ "-host", "localhost:9160"
+ }, false );
+
+ System.out.println("completed");
+ }
+
+ private static int getFileCount(File exportDir, final String ext ) {
+ return exportDir.listFiles( new FileFilter() {
+ @Override
+ public boolean accept(File pathname) {
+ return pathname.getAbsolutePath().endsWith("." + ext);
+ }
+ } ).length;
+ }
+}
+
[25/39] usergrid git commit: Merge branch 'USERGRID-1076' of
https://git-wip-us.apache.org/repos/asf/usergrid into
readRepairForIndexValues
Posted by sn...@apache.org.
Merge branch 'USERGRID-1076' of https://git-wip-us.apache.org/repos/asf/usergrid into readRepairForIndexValues
* 'USERGRID-1076' of https://git-wip-us.apache.org/repos/asf/usergrid:
Added prints for values that end up getting cleaned.
Added ignore to test for use case that doesn't currently work. Added ability to filter out multiple entities with the same user name ( pick the one one modified last )
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/00171f07
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/00171f07
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/00171f07
Branch: refs/heads/1.x
Commit: 00171f07b12a6779f015931d718e30bdf738f11f
Parents: a670579 1fab1db
Author: George Reyes <gr...@apache.org>
Authored: Tue Nov 10 14:15:51 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Tue Nov 10 14:15:51 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 106 +++++++++++++------
.../usergrid/tools/UniqueIndexCleanupTest.java | 2 +-
2 files changed, 75 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
[11/39] usergrid git commit: RC1 for unique index cleanup tool
Posted by sn...@apache.org.
RC1 for unique index cleanup tool
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/03a123dc
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/03a123dc
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/03a123dc
Branch: refs/heads/1.x
Commit: 03a123dc830908f0f1eb46504d39d0050c8887c5
Parents: 9482699
Author: George Reyes <gr...@apache.org>
Authored: Wed Nov 4 16:11:46 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Wed Nov 4 16:11:46 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 24 ++++++++++++++------
1 file changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/03a123dc/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index ba7441b..52288e4 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -83,6 +83,7 @@ import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNI
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraService.APPLICATIONS_CF;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.persistence.cassandra.Serializers.dce;
@@ -204,7 +205,7 @@ public class UniqueIndexCleanup extends ToolBase {
RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery = HFactory
.createRangeSlicesQuery( ko, be, be, be )
.setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
- //not sure if I trust the lower two ssettings as it might iterfere with paging or set arbitrary limits and what I want to retrieve.
+ //not sure if I trust the lower two settings as it might iterfere with paging or set arbitrary limits and what I want to retrieve.
//That needs to be verified.
.setKeys( null, null )
.setRange( null, null, false, 100 );
@@ -217,15 +218,14 @@ public class UniqueIndexCleanup extends ToolBase {
result.get().getList().get( 0 ).getColumnSlice();
while(rangeSlicesIterator.hasNext()) {
- //UUID returned_uuid = UUID.nameUUIDFromBytes(((ByteBuffer)rangeSlicesIterator.next().getKey()).array());
Row rangeSliceValue = rangeSlicesIterator.next();
String returnedRowKey =
new String( ( ( ByteBuffer ) rangeSliceValue.getKey() ).array(), Charsets.UTF_8 ).trim();
String[] parsedRowKey = returnedRowKey.split( ":" );
- if ( parsedRowKey[1].equals( "users" ) || returnedRowKey.contains( "username" ) || returnedRowKey
- .contains( "email" ) ) {
+ if ( parsedRowKey[1].equals( "users" ) ) {
+
ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
if ( columnSlice.getColumns().size() != 0 ) {
System.out.println( returnedRowKey );
@@ -234,13 +234,23 @@ public class UniqueIndexCleanup extends ToolBase {
for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
UUID entityId = ue.fromByteBuffer( col.getName() );
+ if(parsedRowKey[0].equals( MANAGEMENT_APPLICATION_ID.toString() )){
+ if(managementService.getAdminUserByUuid( entityId )==null ){
+ logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
+ System.out.println( "Deleting column uuid: " + entityId.toString() );
+
- if ( em.get( entityId ) == null && managementService.getAdminUserByUuid( entityId )==null ) {
+ Object key = key( applicationId, collectionName, parsedRowKey[2], parsedRowKey[3]);
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
+ m.execute();
+ continue;
+ }
+ }
+ else if ( em.get( entityId ) == null ) {
logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
System.out.println( "Deleting column uuid: " + entityId.toString() );
-
- Object key = key( applicationId, collectionName, "username", entityId );
+ Object key = key( applicationId, collectionName, parsedRowKey[2], parsedRowKey[3]);
addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp );
m.execute();
continue;
[07/39] usergrid git commit: initial steps towards a fix where we
only iterate through the entity_unique column family.
Posted by sn...@apache.org.
initial steps towards a fix where we only iterate through the entity_unique column family.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5dabe6e0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5dabe6e0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5dabe6e0
Branch: refs/heads/1.x
Commit: 5dabe6e07159a684d6e5c6f5dd57353b3d2e79fe
Parents: a4a7884
Author: George Reyes <gr...@apache.org>
Authored: Mon Nov 2 15:30:10 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Mon Nov 2 15:30:10 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 156 +++++++++++--------
1 file changed, 89 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5dabe6e0/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index bf8f0f2..f86da7b 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -181,7 +181,7 @@ public class UniqueIndexCleanup extends ToolBase {
for ( final String bucketName : indexBucketLocator.getBuckets() ) {
IndexScanner scanner =
- cass.getIdList( key( applicationId, DICTIONARY_COLLECTIONS, collectionName ), null, null,
+ cass.getIdList( key( applicationId, ENTITY_UNIQUE, collectionName ), null, null,
PAGE_SIZE, false, bucketName, applicationId, false );
SliceIterator itr = new SliceIterator( scanner, new UUIDIndexSliceParser( null ) );
@@ -211,73 +211,93 @@ public class UniqueIndexCleanup extends ToolBase {
try {
- for ( String prop : indexed ) {
-
- String bucket = indexBucketLocator
- .getBucket( id );
-
- Object rowKey = key( applicationId, collection.getName(), prop, bucket );
-
- List<HColumn<ByteBuffer, ByteBuffer>> indexCols =
- scanIndexForAllTypes( ko, indexBucketLocator, applicationId, rowKey, id,
- prop );
-
- // loop through the indexed values and verify them as present in
- // our entity_index_entries. If they aren't, we need to delete the
- // from the secondary index, and mark
- // this object for re-index via n update
- for ( HColumn<ByteBuffer, ByteBuffer> index : indexCols ) {
-
- DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( index
- .getName().duplicate() );
-
- Object code = secondaryIndexValue.get( 0 );
- Object propValue = secondaryIndexValue.get( 1 );
- UUID timestampId = ( UUID ) secondaryIndexValue.get( 3 );
-
- DynamicComposite existingEntryStart = new DynamicComposite( prop, code,
- propValue, timestampId );
- DynamicComposite existingEntryFinish = new DynamicComposite( prop, code,
- propValue, timestampId );
-
- setEqualityFlag( existingEntryFinish, ComponentEquality.GREATER_THAN_EQUAL );
-
- // now search our EntityIndexEntry for previous values, see if
- // they don't match this one
-
- List<HColumn<ByteBuffer, ByteBuffer>> entries =
- cass.getColumns( ko, ENTITY_INDEX_ENTRIES, id, existingEntryStart,
- existingEntryFinish, INDEX_ENTRY_LIST_COUNT, false );
-
- // we wouldn't find this column in our entity_index_entries
- // audit. Delete it, then mark this entity for update
- if ( entries.size() == 0 ) {
- logger.info(
- "Could not find reference to value '{}' for property '{}' on entity "
- +
- "{} in collection {}. " + " Forcing reindex", new Object[] { propValue, prop, id, collectionName } );
- Object key = key( applicationId, collectionName, prop, id );
- addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
-
- addDeleteToMutator( m, ENTITY_INDEX, rowKey, index.getName().duplicate(),
- timestamp );
-
- reIndex = true;
- }
-
- if ( entries.size() > 1 ) {
- Object key = key( applicationId, collectionName, prop, id );
- addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
- logger.info(
- "Found more than 1 entity referencing unique index for property "
- + "'{}' "
- +
- "with value " + "'{}'", prop, propValue );
- reIndex = true;
- }
- }
+ Entity entity = em.get( id );
+
+ //entity may not exist, but we should have deleted rows from the index
+ if ( entity == null ) {
+ logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
+ //now execute the cleanup. In this case the entity is gone,
+ // so we'll want to remove references from
+ // the secondary index also remove from unique entity.
+ Object key = key( applicationId, collectionName,"username", id );
+ addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
+ m.execute();
+ continue;
}
+ logger.info( "Reindex complete for entity with id '{} ", id );
+
+ //now execute the cleanup. This way if the above update fails,
+ // we still have enough data to run again
+ // later
+ m.execute();
+
+//
+// for ( String prop : indexed ) {
+//
+// String bucket = indexBucketLocator
+// .getBucket( id );
+//
+// Object rowKey = key( applicationId, collection.getName(), prop, bucket );
+//
+// List<HColumn<ByteBuffer, ByteBuffer>> indexCols =
+// scanIndexForAllTypes( ko, indexBucketLocator, applicationId, rowKey, id,
+// prop );
+//
+// // loop through the indexed values and verify them as present in
+// // our entity_index_entries. If they aren't, we need to delete the
+// // from the secondary index, and mark
+// // this object for re-index via n update
+// for ( HColumn<ByteBuffer, ByteBuffer> index : indexCols ) {
+//
+// DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( index
+// .getName().duplicate() );
+//
+// Object code = secondaryIndexValue.get( 0 );
+// Object propValue = secondaryIndexValue.get( 1 );
+// UUID timestampId = ( UUID ) secondaryIndexValue.get( 3 );
+//
+// DynamicComposite existingEntryStart = new DynamicComposite( prop, code,
+// propValue, timestampId );
+// DynamicComposite existingEntryFinish = new DynamicComposite( prop, code,
+// propValue, timestampId );
+//
+// setEqualityFlag( existingEntryFinish, ComponentEquality.GREATER_THAN_EQUAL );
+//
+// // now search our EntityIndexEntry for previous values, see if
+// // they don't match this one
+//
+// List<HColumn<ByteBuffer, ByteBuffer>> entries =
+// cass.getColumns( ko, ENTITY_INDEX_ENTRIES, id, existingEntryStart,
+// existingEntryFinish, INDEX_ENTRY_LIST_COUNT, false );
+//
+// // we wouldn't find this column in our entity_index_entries
+// // audit. Delete it, then mark this entity for update
+// if ( entries.size() == 0 ) {
+// logger.info(
+// "Could not find reference to value '{}' for property '{}' on entity "
+// +
+// "{} in collection {}. " + " Forcing reindex", new Object[] { propValue, prop, id, collectionName } );
+//
+// addDeleteToMutator( m, ENTITY_INDEX, rowKey, index.getName().duplicate(),
+// timestamp );
+//
+// reIndex = true;
+// }
+//
+// if ( entries.size() > 1 ) {
+// Object key = key( applicationId, collectionName, prop, id );
+// addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, id );
+// logger.info(
+// "Found more than 1 entity referencing unique index for property "
+// + "'{}' "
+// +
+// "with value " + "'{}'", prop, propValue );
+// reIndex = true;
+// }
+// }
+// }
+
//force this entity to be updated
if ( reIndex ) {
Entity entity = em.get( id );
@@ -287,7 +307,9 @@ public class UniqueIndexCleanup extends ToolBase {
logger.warn( "Entity with id {} did not exist in app {}", id, applicationId );
//now execute the cleanup. In this case the entity is gone,
// so we'll want to remove references from
- // the secondary index
+ // the secondary index also remove from unique entity.
+
+
m.execute();
continue;
}
[22/39] usergrid git commit: Added tests and read repair to the guts
of 1.0
Posted by sn...@apache.org.
Added tests and read repair to the guts of 1.0
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a6705790
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a6705790
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a6705790
Branch: refs/heads/1.x
Commit: a67057909e480c97273c9f6c366643069f24e114
Parents: 7db5769
Author: George Reyes <gr...@apache.org>
Authored: Tue Nov 10 09:06:15 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Tue Nov 10 09:06:15 2015 -0800
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 32 ++++
.../usergrid/persistence/EntityManagerIT.java | 2 +
.../usergrid/services/ServiceRequestIT.java | 169 ++++++++++++++++++-
3 files changed, 202 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6705790/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index 4a6e2ea..ecd0f0a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -109,6 +109,7 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static java.util.Arrays.asList;
import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery;
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
import static org.apache.commons.lang.StringUtils.capitalize;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.usergrid.locking.LockHelper.getUniqueUpdateLock;
@@ -156,6 +157,7 @@ import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtil
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.toStorableBinaryValue;
import static org.apache.usergrid.persistence.cassandra.CassandraService.ALL_COUNT;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.persistence.cassandra.Serializers.le;
import static org.apache.usergrid.persistence.cassandra.Serializers.se;
@@ -538,6 +540,8 @@ public class EntityManagerImpl implements EntityManager {
Object key = createUniqueIndexKey( ownerEntityId, collectionNameInternal, propertyName, propertyValue );
+ //need to fix by asking todd as to why 2.
+ //why is this set to 2?
List<HColumn<ByteBuffer, ByteBuffer>> cols =
cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
false );
@@ -549,12 +553,40 @@ public class EntityManagerImpl implements EntityManager {
}
//shouldn't happen, but it's an error case
+
if ( cols.size() > 1 ) {
logger.error( "INDEX CORRUPTION: More than 1 unique value exists for entities in ownerId {} of type {} on "
+ "property {} with value {}",
new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } );
+
+ //retreive up to 100 columns
+ List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 100,
+ false );
+
+
+
+ //go through it column
+ for ( HColumn<ByteBuffer, ByteBuffer> col : indexingColumns ) {
+ UUID indexCorruptionUuid = ue.fromByteBuffer( col.getName());
+ if (get( indexCorruptionUuid ) == null ) {
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+ Keyspace ko = cass.getApplicationKeyspace( ownerEntityId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, indexCorruptionUuid, timestamp );
+ mutator.execute();
+ cols.remove( col );
+ }
+ else{
+
+ }
+ }
}
+ cols = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
+ false );
+
/**
* Doing this in a loop sucks, but we need to account for possibly having more than 1 entry in the index due
* to corruption. We need to allow them to update, otherwise
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6705790/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
index 5cfc013..b911b3b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java
@@ -561,4 +561,6 @@ public class EntityManagerIT extends AbstractCoreIT {
//Not an owner
assertFalse( em.isCollectionMember( createdUser2, "devices", createdDevice ) );
}
+
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6705790/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
index 5b5428a..d89536f 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java
@@ -17,7 +17,9 @@
package org.apache.usergrid.services;
+import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -27,15 +29,39 @@ import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.RandomStringUtils;
+
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.ServiceITSuite;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.cassandra.Concurrent;
-
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationOwnerInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.mutation.Mutator;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.services.ServiceParameter.filter;
import static org.apache.usergrid.services.ServiceParameter.parameters;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
@Concurrent()
@@ -76,4 +102,145 @@ public class ServiceRequestIT {
p = filter( path.getParameters(), replaceParameters );
logger.info( "" + p );
}
+
+ //Verify that entity read repair is functioning as intended.
+ @Test
+ public void testRepairOfSingleEntity() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ fail("shouldn't throw something else i think");
+ }
+
+
+ entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ assertNotNull( entityToBeCorrupted );
+ assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername() );
+
+ }
+
+
+ @Test
+ public void testRepairOfOnlyOneOfTwoColumns() throws Exception{
+ String rand = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String orgName = "org_" + rand;
+ String appName = "app_" +rand;
+ String username = "username_" + rand;
+ String adminUsername = "admin_"+rand;
+ String email = username+"@derp.com";
+ String password = username;
+
+ String collectionName = "users";
+
+
+ OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
+
+ ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
+
+ EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
+
+ Map<String,Object> userInfo = new HashMap<String, Object>( );
+ userInfo.put( "username",username );
+
+ //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
+
+ CassandraService cass = setup.getCassSvc();
+
+ Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ //create a new column
+ Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
+
+ UUID testEntityUUID = UUIDUtils.newTimeUUID();
+ //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
+ addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
+
+ m.execute();
+
+ //verify that there is no corresponding entity with the uuid or alias provided
+ //verify it returns null.
+ assertNull(entityManager.get( testEntityUUID ));
+
+ //the below works but not needed for this test.
+ //assertNull( entityManager.getAlias("user",username));
+
+ //verify that we cannot recreate the entity due to duplicate unique property exception
+ Entity entityToBeCorrupted = null;
+ try {
+ entityToBeCorrupted = entityManager.create( collectionName, userInfo );
+ fail();
+ }catch(DuplicateUniquePropertyExistsException dup){
+
+ }
+ catch(Exception e){
+ throw e;
+ }
+
+ //should return null since we have duplicate alias. Will Cause index corruptions to be thrown.
+ //TODO: fix the below so that it fails everytime.
+ //50/50 chance to succeed or fail
+ // assertNull( entityManager
+ // .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+
+
+ //verifies it works now.
+ assertNotNull( entityManager
+ .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
+
+ }
}
[36/39] usergrid git commit: Added instructions on how to
runUserUniqueIndexCleanup. Replaced UniqueIndexCleanup with
UserUniqueIndexCleanup as it only cleans up users. Changed all
system.out.prints to logger type messages
Posted by sn...@apache.org.
Added instructions on how to runUserUniqueIndexCleanup.
Replaced UniqueIndexCleanup with UserUniqueIndexCleanup as it only cleans up users.
Changed all system.out.prints to logger type messages
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/15e25dea
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/15e25dea
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/15e25dea
Branch: refs/heads/1.x
Commit: 15e25dea784257988bbc46a3b3e0185ec09aca23
Parents: a874aa9
Author: George Reyes <gr...@apache.org>
Authored: Fri Nov 13 08:59:14 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Fri Nov 13 08:59:14 2015 -0800
----------------------------------------------------------------------
.../cassandra/EntityManagerImpl.java | 18 +-
.../usergrid/tools/UniqueIndexCleanup.java | 473 ------------------
.../usergrid/tools/UserUniqueIndexCleanup.java | 478 ++++++++++++++++++
.../usergrid/tools/UniqueIndexCleanupTest.java | 495 -------------------
.../tools/UserUniqueIndexCleanupTest.java | 490 ++++++++++++++++++
5 files changed, 981 insertions(+), 973 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index ea8c28e..80d9f44 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -571,11 +571,12 @@ public class EntityManagerImpl implements EntityManager {
+ "property {} with value {}",
new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } );
- //retrieve up to 100 columns
- List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 100,
- false );
+ //retrieve ALL DA columns.
+ List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getAllColumns( cass.getApplicationKeyspace( applicationId ),ENTITY_UNIQUE,key,be,be );
+ //Contains entities that weren't deleted but are still in index.
+ //maybe use a set or list so you don't have to keep track of an index.
Entity[] entities = new Entity[cols.size()];
int index = 0;
@@ -586,7 +587,8 @@ public class EntityManagerImpl implements EntityManager {
if (entities[index] == null ) {
deleteUniqueColumn( ownerEntityId, key, indexCorruptionUuid );
- cols.remove( col );
+ //iffy since cols won't have the same values of indexingColumns.
+ //cols.remove( col );
}
else{
index++;
@@ -607,7 +609,7 @@ public class EntityManagerImpl implements EntityManager {
mostRecentEntity = entity;
}
else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
- logger.info("Entities with unique value: "+propertyValue+" has two or more entities with the same modified time."
+ logger.error("Entities with unique value: "+propertyValue+" has two or more entities with the same modified time."
+ "Please manually resolve by query or changing names. ");
}
}
@@ -617,6 +619,12 @@ public class EntityManagerImpl implements EntityManager {
cols = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2,
false );
+ if ( cols.size() > 1 ) {
+ logger.error( "READ REPAIR FAILURE: More than 1 unique value still exists for entities in ownerId {} of type {} on "
+ + "property {} with value {}",
+ new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } );
+ }
+
/**
* Doing this in a loop sucks, but we need to account for possibly having more than 1 entry in the index due
* to corruption. We need to allow them to update, otherwise
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
deleted file mode 100644
index 64b634c..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ /dev/null
@@ -1,473 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.cassandra.thrift.TimedOutException;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.thrift.TBaseHelper;
-
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
-import org.apache.usergrid.utils.StringUtils;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import me.prettyprint.cassandra.service.RangeSlicesIterator;
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.ColumnSlice;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.beans.Row;
-import me.prettyprint.hector.api.factory.HFactory;
-import me.prettyprint.hector.api.mutation.Mutator;
-import me.prettyprint.hector.api.query.RangeSlicesQuery;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
-import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-
-
-/**
- * This is a utility to audit all available entity ids in the secondary index. It then checks to see if any index value
- * is not present in the Entity_Index_Entries. If it is not, the value from the index is removed, and a forced re-index
- * is triggered <p/> USERGRID-323 <p/> <p/> UniqueIndexCleanup -app [appid] -col [collectionname]
- *
- * @author tnine
- */
-public class UniqueIndexCleanup extends ToolBase {
-
- /**
- *
- */
- private static final int PAGE_SIZE = 100;
-
-
- private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
-
- private static final String APPLICATION_ARG = "app";
-
- private static final String COLLECTION_ARG = "col";
-
- private static final String ENTITY_UNIQUE_PROPERTY_NAME = "property";
-
- private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
-
-
- @Override
- @SuppressWarnings( "static-access" )
- public Options createOptions() {
-
-
- Options options = new Options();
-
- Option hostOption =
- OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
- .create( "host" );
-
- options.addOption( hostOption );
-
-
- Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
- .withDescription( "application id" ).create( APPLICATION_ARG );
-
-
- options.addOption( appOption );
-
- Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
- .withDescription( "collection name" ).create( COLLECTION_ARG );
-
- options.addOption( collectionOption );
-
- Option entityUniquePropertyName =
- OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_NAME ).hasArg().isRequired( false )
- .withDescription( "Entity Unique Property Name" ).create( ENTITY_UNIQUE_PROPERTY_NAME );
- options.addOption( entityUniquePropertyName );
-
- Option entityUniquePropertyValue =
- OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
- .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
- options.addOption( entityUniquePropertyValue );
-
-
- return options;
- }
-
-
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
- */
- @Override
- public void runTool( CommandLine line ) throws Exception {
- startSpring();
-
- logger.info( "Starting entity unique index cleanup" );
-
-
- // go through each collection and audit the values
- Keyspace ko = cass.getUsergridApplicationKeyspace();
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- if ( line.hasOption( APPLICATION_ARG ) || line.hasOption( COLLECTION_ARG ) ||
- line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
- deleteInvalidValuesForUniqueProperty( m, line );
- }
- else {
-
- RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
- HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
- //not sure if I trust the lower two settings as it might iterfere with paging or set
- // arbitrary limits and what I want to retrieve.
- //That needs to be verified.
- .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
-
-
- RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
-
- while ( rangeSlicesIterator.hasNext() ) {
- Row rangeSliceValue = rangeSlicesIterator.next();
-
-
- ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
- //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
- // ones return garbage data.
- //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
- String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
- Charset.defaultCharset() ).trim();
-
-
- //defensive programming, don't have to have to parse the string if it doesn't contain users.
- if ( returnedRowKey.contains( "users" ) ) {
-
- String[] parsedRowKey = returnedRowKey.split( ":" );
-
- //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
- if ( parsedRowKey.length > 4 ) {
- parsedRowKey = garbageRowKeyParser( parsedRowKey );
-
- if ( parsedRowKey == null ) {
- System.out.println( returnedRowKey + " is a invalid row key, and unparseable. Skipped..." );
- continue;
- }
- }
- //if the rowkey contains less than four parts then it is completely invalid
- else if ( parsedRowKey.length < 4 ) {
- System.out.println( returnedRowKey + " is a invalid row key and will be skipped" );
- continue;
- }
-
- UUID applicationId = null;
- try {
- applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
- }
- catch ( Exception e ) {
- continue;
- }
- String collectionName = parsedRowKey[1];
- String uniqueValueKey = parsedRowKey[2];
- String uniqueValue = parsedRowKey[3];
-
-
- if ( collectionName.equals( "users" ) ) {
-
- ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
- if ( columnSlice.getColumns().size() != 0 ) {
- List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
- if ( cols.size() == 0 ) {
- deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
- }
- else {
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,returnedRowKey );
- }
- }
- }
- }
- }
- }
- System.out.println( "Completed repair." );
-
- logger.info( "Completed audit of apps" );
- }
-
-
- //Returns a functioning rowkey if it can otherwise returns null
- public String[] garbageRowKeyParser( String[] parsedRowKey ) {
- String[] modifiedRowKey = parsedRowKey.clone();
- while ( modifiedRowKey != null ) {
- if ( modifiedRowKey.length < 4 ) {
- return null;
- }
-
- String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
- if ( recreatedRowKey == null ) {
- recreatedRowKey = "";
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- }
- else {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
- break;
- }
- }
- return modifiedRowKey;
- }
-
-
- private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
- for ( int i = 1; i < modifiedRowKey.length; i++ ) {
-
- recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
- if ( i + 1 != modifiedRowKey.length ) {
- recreatedRowKey = recreatedRowKey.concat( ":" );
- }
- }
- modifiedRowKey = recreatedRowKey.split( ":" );
- return modifiedRowKey;
- }
-
-
- private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue ) throws Exception {
- System.out.println( "Found 0 uuid's associated with: " + uniqueValue );
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationId );
- Mutator<ByteBuffer> mutator = createMutator( ko, be );
-
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( mutator, ENTITY_UNIQUE, key, timestamp );
- mutator.execute();
- return;
- }
-
-
- private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue,
- final List<HColumn<ByteBuffer, ByteBuffer>> cols, String rowKey ) throws Exception {
- Boolean cleanup = false;
- EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
- int numberOfColumnsDeleted = 0;
- //these columns all come from the same row key, which means they each belong to the same row key identifier
- //thus mixing and matching them in the below if cases won't matter.
- Entity[] entities = new Entity[cols.size()];
- int numberOfRetrys = 5;
-
- if(cols.size() < 2){
- entities = new Entity[2];
- }
-
- int index = 0;
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- UUID entityId = ue.fromByteBuffer( col.getName() );
-
- if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
- for(int i = 0; i<numberOfRetrys; i++){
- try {
- entities[index] = managementService.getAdminUserEntityByUuid( entityId );
- break;
- }catch(TimedOutException toe){
- Thread.sleep( 2000 );
- }
- }
-
- if ( entities[index] == null ) {
- cleanup = true;
- }
- else {
- index++;
- }
- }
- else {
-
- for(int i = 0; i<numberOfRetrys; i++){
- try {
- entities[index] = em.get( entityId );
- break;
- }catch(TimedOutException toe){
- Thread.sleep( 2000 );
- }
- }
- if ( entities[index] == null ) {
- cleanup = true;
- }
- else {
- index++;
- }
- }
-
- if ( cleanup == true ) {
- numberOfColumnsDeleted++;
- deleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue, entityId );
- cleanup = false;
- }
- }
-
- //this means that the same unique rowkey has two values associated with it
- if(index>2){
- Entity mostRecentEntity = entities[0];
- for(Entity entity: entities){
- if(mostRecentEntity == null){
- System.out.println( "Most Recent entity is null and is being replaced by regular entity" );
- mostRecentEntity = entity;
- }
- if(entity == null){
- System.out.println("Entity we're cycling through is null and we need more new entities");
- continue;
- }
-
- mostRecentEntity = verifyModifiedTimestamp( mostRecentEntity );
- entity = verifyModifiedTimestamp( entity );
-
-
- if(mostRecentEntity.getModified() > entity.getModified()){
- System.out.println("Deleting "+entity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
- System.out.flush();
- em.deleteEntity( entity.getUuid() );
-
- }
- else if (mostRecentEntity.getModified() < entity.getModified()){
- System.out.println("Deleting "+mostRecentEntity.getUuid().toString()+" because it is the older column in the following rowkey: "+rowKey);
- System.out.flush();
- em.deleteEntity( mostRecentEntity.getUuid() );
- mostRecentEntity = entity;
- }
- else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
- System.out.println("Entities with unique value: "+uniqueValue+" has two or more entities with the same modified time."
- + "Please manually resolve by query or changing names. ");
- }
- }
- }
-
-
- //a safer way to do this would be to try to do another get and verify there is nothing left in the column
- //instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
- if ( cols.size() == numberOfColumnsDeleted ) {
- deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
- }
- }
-
-
- private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
- Entity entity = unverifiedEntity;
- if(entity !=null && entity.getModified()==null) {
- if(entity.getCreated()!=null){
- System.out.println("Setting modified timestamp to created for comparison purposes");
- entity.setModified( entity.getCreated() );
- return entity;
- }
- else{
- System.out.println("Please delete or remake: "+entity.getUuid());
- System.out.println("Setting timestamps to 1");
- entity.setCreated( 1L );
- entity.setModified( 1L );
- return entity;
- }
- }
- return entity;
- }
-
-
- //really only deletes ones that aren't existant for a specific value
- private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
- UUID applicationId = UUID.fromString( line.getOptionValue( APPLICATION_ARG ) );
- String collectionName = line.getOptionValue( COLLECTION_ARG );
- String uniqueValueKey = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
- String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
-
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
-
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols =
- cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
- false );
-
-
- if ( cols.size() == 0 ) {
- System.out.println( "Zero columns were found for " + key.toString() + ". Will delete rowkey." );
- }
-
- entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,key.toString() );
- }
-
-
- private String uuidGarbageParser( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- System.out.println( garbageString + " is unparsable" );
- break;
- }
- }
- return stringToBeTruncated;
- }
-
-
- private String uuidStringVerifier( final String garbageString ) {
- int index = 1;
- String stringToBeTruncated = garbageString;
- while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
- if ( stringToBeTruncated.length() > 36 ) {
- stringToBeTruncated = stringToBeTruncated.substring( index );
- }
- else {
- return null;
- }
- }
- return stringToBeTruncated;
- }
-
-
- private void deleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
- final String uniqueValueKey, final String uniqueValue, final UUID entityId )
- throws Exception {
- logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId );
- System.out.println( "Deleting column uuid: " + entityId.toString() );
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
- Keyspace ko = cass.getApplicationKeyspace( applicationId );
- Mutator<ByteBuffer> mutator = createMutator( ko, be );
-
- Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
- addDeleteToMutator( mutator, ENTITY_UNIQUE, key, entityId, timestamp );
- mutator.execute();
- return;
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
new file mode 100644
index 0000000..d95a6e7
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserUniqueIndexCleanup.java
@@ -0,0 +1,478 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.cassandra.thrift.TimedOutException;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.thrift.TBaseHelper;
+
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import me.prettyprint.cassandra.service.RangeSlicesIterator;
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.ColumnSlice;
+import me.prettyprint.hector.api.beans.HColumn;
+import me.prettyprint.hector.api.beans.Row;
+import me.prettyprint.hector.api.factory.HFactory;
+import me.prettyprint.hector.api.mutation.Mutator;
+import me.prettyprint.hector.api.query.RangeSlicesQuery;
+
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
+import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
+import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
+
+//TODO: write docs so support can run it.
+//TODO: provide support with log4j file and instructions on how to use it.
+/**
+ *
+ *This utility audits all values in the ENTITY_UNIQUE column family. If it finds any duplicates of users
+ * then it deletes the non existing columns from the row. If there are no more columns in the row then it deletes the row.
+ * If there exists more than one existing column then the one with the most recent timestamp wins and the other is deleted.
+ *
+ *If you want the run the tool on their cluster the following is what you need to do
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here> > log.txt
+ *
+ * if there is a specific value you want to run the tool on then you need the following
+
+ * nohup java -Dlog4j.configuration=file:log4j.properties -jar usergrid-tools-1.0.2.jar UniqueIndexCleanup -host <cassandra_host_here>
+ * -app <applicationUUID> -col <collection_name> -property <unique_property_key> -value <unique_property_value> > log.txt
+ *
+ * @author grey
+ */
+public class UserUniqueIndexCleanup extends ToolBase {
+
+ /**
+ *
+ */
+ private static final int PAGE_SIZE = 100;
+
+
+ private static final Logger logger = LoggerFactory.getLogger( UserUniqueIndexCleanup.class );
+
+ private static final String APPLICATION_ARG = "app";
+
+ private static final String COLLECTION_ARG = "col";
+
+ private static final String ENTITY_UNIQUE_PROPERTY_NAME = "property";
+
+ private static final String ENTITY_UNIQUE_PROPERTY_VALUE = "value";
+
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+
+ Options options = new Options();
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ options.addOption( hostOption );
+
+
+ Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "application id" ).create( APPLICATION_ARG );
+
+
+ options.addOption( appOption );
+
+ Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
+ .withDescription( "collection name" ).create( COLLECTION_ARG );
+
+ options.addOption( collectionOption );
+
+ Option entityUniquePropertyName =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_NAME ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Name" ).create( ENTITY_UNIQUE_PROPERTY_NAME );
+ options.addOption( entityUniquePropertyName );
+
+ Option entityUniquePropertyValue =
+ OptionBuilder.withArgName( ENTITY_UNIQUE_PROPERTY_VALUE ).hasArg().isRequired( false )
+ .withDescription( "Entity Unique Property Value" ).create( ENTITY_UNIQUE_PROPERTY_VALUE );
+ options.addOption( entityUniquePropertyValue );
+
+
+ return options;
+ }
+
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.usergrid.tools.ToolBase#runTool(org.apache.commons.cli.CommandLine)
+ */
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+ startSpring();
+
+ logger.info( "Starting entity unique index cleanup" );
+
+
+ // go through each collection and audit the values
+ Keyspace ko = cass.getUsergridApplicationKeyspace();
+ Mutator<ByteBuffer> m = createMutator( ko, be );
+
+ if ( line.hasOption( APPLICATION_ARG ) || line.hasOption( COLLECTION_ARG ) ||
+ line.hasOption( ENTITY_UNIQUE_PROPERTY_NAME ) || line.hasOption( ENTITY_UNIQUE_PROPERTY_VALUE ) ) {
+ deleteInvalidValuesForUniqueProperty( m, line );
+ }
+ else {
+//maybe put a byte buffer infront.
+ RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery =
+ HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() )
+ //not sure if I trust the lower two settings as it might iterfere with paging or set
+ // arbitrary limits and what I want to retrieve.
+ //That needs to be verified.
+ .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
+
+
+ RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
+
+ while ( rangeSlicesIterator.hasNext() ) {
+ Row rangeSliceValue = rangeSlicesIterator.next();
+
+
+ ByteBuffer buf = ( TBaseHelper.rightSize( ( ByteBuffer ) rangeSliceValue.getKey() ) );
+ //Cassandra client library returns ByteBuffers that are views on top of a larger byte[]. These larger
+ // ones return garbage data.
+ //Discovered thanks due to https://issues.apache.org/jira/browse/NUTCH-1591
+ String returnedRowKey = new String( buf.array(), buf.arrayOffset() + buf.position(), buf.remaining(),
+ Charset.defaultCharset() ).trim();
+
+
+ //defensive programming, don't have to have to parse the string if it doesn't contain users.
+ if ( returnedRowKey.contains( "users" ) ) {
+
+ String[] parsedRowKey = returnedRowKey.split( ":" );
+
+ //if the rowkey contains more than 4 parts then it may have some garbage appended to the front.
+ if ( parsedRowKey.length > 4 ) {
+ parsedRowKey = garbageRowKeyParser( parsedRowKey );
+
+ if ( parsedRowKey == null ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...",returnedRowKey );
+ continue;
+ }
+ }
+ //if the rowkey contains less than four parts then it is completely invalid
+ else if ( parsedRowKey.length < 4 ) {
+ logger.error( "{} is a invalid row key, and unparseable. Skipped...",returnedRowKey );
+ continue;
+ }
+
+ UUID applicationId = null;
+ try {
+ applicationId = UUID.fromString( uuidGarbageParser( parsedRowKey[0] ) );
+ }
+ catch ( Exception e ) {
+ logger.error( "could not parse {} despite earlier parsing. Skipping...",parsedRowKey[0] );
+ continue;
+ }
+ String collectionName = parsedRowKey[1];
+ String uniqueValueKey = parsedRowKey[2];
+ String uniqueValue = parsedRowKey[3];
+
+
+ if ( collectionName.equals( "users" ) ) {
+
+ ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice();
+ //if ( columnSlice.getColumns().size() != 0 ) {
+ List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns();
+ if ( cols.size() == 0 ) {
+ deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
+ }
+ else {
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,returnedRowKey );
+ }
+ // }
+ }
+ }
+ }
+ }
+ logger.info( "Completed repair successfully" );
+ }
+
+
+ //Returns a functioning rowkey if it can otherwise returns null
+ public String[] garbageRowKeyParser( String[] parsedRowKey ) {
+ String[] modifiedRowKey = parsedRowKey.clone();
+ while ( modifiedRowKey != null ) {
+ if ( modifiedRowKey.length < 4 ) {
+ return null;
+ }
+
+ String recreatedRowKey = uuidStringVerifier( modifiedRowKey[0] );
+ if ( recreatedRowKey == null ) {
+ recreatedRowKey = "";
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ }
+ else {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ modifiedRowKey = getStrings( modifiedRowKey, recreatedRowKey );
+ break;
+ }
+ }
+ return modifiedRowKey;
+ }
+
+
+ private String[] getStrings( String[] modifiedRowKey, String recreatedRowKey ) {
+ for ( int i = 1; i < modifiedRowKey.length; i++ ) {
+
+ recreatedRowKey = recreatedRowKey.concat( modifiedRowKey[i] );
+ if ( i + 1 != modifiedRowKey.length ) {
+ recreatedRowKey = recreatedRowKey.concat( ":" );
+ }
+ }
+ modifiedRowKey = recreatedRowKey.split( ":" );
+ return modifiedRowKey;
+ }
+
+
+ private void deleteRow( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue ) throws Exception {
+ logger.debug( "Found 0 uuid's associated with {} Deleting row.",uniqueValue );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, timestamp );
+ mutator.execute();
+ return;
+ }
+
+
+ private void entityUUIDDelete( final Mutator<ByteBuffer> m, final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue,
+ final List<HColumn<ByteBuffer, ByteBuffer>> cols, String rowKey ) throws Exception {
+ Boolean cleanup = false;
+ EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId );
+ int numberOfColumnsDeleted = 0;
+ //these columns all come from the same row key, which means they each belong to the same row key identifier
+ //thus mixing and matching them in the below if cases won't matter.
+ Entity[] entities = new Entity[cols.size()];
+ int numberOfRetrys = 5;
+ int numberOfTimesRetrying = 0;
+
+ int index = 0;
+
+ for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
+ UUID entityId = ue.fromByteBuffer( col.getName() );
+
+ //could be the same and just do a entity get
+ if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) {
+ for(numberOfTimesRetrying = 0; numberOfTimesRetrying<numberOfRetrys; numberOfTimesRetrying++){
+ try {
+ entities[index] = managementService.getAdminUserEntityByUuid( entityId );
+ break;
+ }catch(TimedOutException toe){
+ Thread.sleep( 2000 );
+ }
+ }
+ if(numberOfTimesRetrying == numberOfRetrys ){
+ logger.error( "Tried {} number of times to get the following uuid: {} but failed.Moving on",numberOfRetrys,entityId );
+ continue;
+ }
+
+ if ( entities[index] == null ) {
+ cleanup = true;
+ }
+ else {
+ index++;
+ }
+ }
+ else {
+
+ for(int i = 0; i<numberOfRetrys; i++){
+ try {
+ entities[index] = em.get( entityId );
+ break;
+ }catch(TimedOutException toe){
+ Thread.sleep( 2000 );
+ }
+ }
+ if(numberOfTimesRetrying == numberOfRetrys ){
+ logger.error( "Tried {} number of times to get the following uuid: {} but failed.Moving on",numberOfRetrys,entityId );
+ continue;
+ }
+
+ if ( entities[index] == null ) {
+ cleanup = true;
+ }
+ else {
+ index++;
+ }
+ }
+
+ if ( cleanup == true ) {
+ numberOfColumnsDeleted++;
+ deleteUniqueValue(applicationId, collectionName, uniqueValueKey, uniqueValue, entityId );
+ cleanup = false;
+ }
+ }
+
+ //this means that the same unique rowkey has two values associated with it
+ if(index>2){
+ Entity mostRecentEntity = entities[0];
+ for(Entity entity: entities){
+
+ mostRecentEntity = verifyModifiedTimestamp( mostRecentEntity );
+ entity = verifyModifiedTimestamp( entity );
+
+
+ if(mostRecentEntity.getModified() > entity.getModified()){
+ logger.error( "Deleting {} because it is the older column in the following rowkey: {} ",entity.getUuid().toString(),rowKey );
+ em.deleteEntity( entity.getUuid() );
+ }
+ else if (mostRecentEntity.getModified() < entity.getModified()){
+ logger.error( "Deleting {} because it is the older column in the following rowkey: {} ",mostRecentEntity.getUuid().toString(),rowKey );
+ em.deleteEntity( mostRecentEntity.getUuid() );
+ mostRecentEntity = entity;
+ }
+ else if (mostRecentEntity.getModified() == entity.getModified() && !mostRecentEntity.getUuid().equals( entity.getUuid() )){
+ logger.error( "Entities with unique value: {} contains two or more entities with the same modified time. Please manually fix these"
+ + "entities by deleting one.",rowKey);
+ }
+ }
+ }
+
+
+ //a safer way to do this would be to try to do another get and verify there is nothing left in the column
+ //instead of just doing a simple check since the column check happens anywhere between 2 to 1000 times.
+ if ( cols.size() == numberOfColumnsDeleted ) {
+ deleteRow( m, applicationId, collectionName, uniqueValueKey, uniqueValue );
+ }
+ }
+
+
+ private Entity verifyModifiedTimestamp( final Entity unverifiedEntity ) {
+ Entity entity = unverifiedEntity;
+ if(entity !=null && entity.getModified()==null) {
+ if(entity.getCreated()!=null){
+ logger.debug("{} has no modified. Subsituting created timestamp for their modified timestamp.Manually adding one for comparison purposes",entity.getUuid());
+ entity.setModified( entity.getCreated() );
+ return entity;
+ }
+ else{
+ logger.error( "Found no created or modified timestamp. Please remake the following entity: {}."
+ + " Setting both created and modified to 1",entity.getUuid().toString() );
+ entity.setCreated( 1L );
+ entity.setModified( 1L );
+ return entity;
+ }
+ }
+ return entity;
+ }
+
+
+ //really only deletes ones that aren't existant for a specific value
+ private void deleteInvalidValuesForUniqueProperty( Mutator<ByteBuffer> m, CommandLine line ) throws Exception {
+ UUID applicationId = UUID.fromString( line.getOptionValue( APPLICATION_ARG ) );
+ String collectionName = line.getOptionValue( COLLECTION_ARG );
+ String uniqueValueKey = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_NAME );
+ String uniqueValue = line.getOptionValue( ENTITY_UNIQUE_PROPERTY_VALUE );
+
+ //PLEASE ADD VERIFICATION.
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+
+
+ List<HColumn<ByteBuffer, ByteBuffer>> cols =
+ cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 1000,
+ false );
+
+
+ if ( cols.size() == 0 ) {
+ logger.error( "This row key: {} has zero columns. Deleting...",key.toString() );
+ }
+
+ entityUUIDDelete( m, applicationId, collectionName, uniqueValueKey, uniqueValue, cols,key.toString() );
+ }
+
+
+ private String uuidGarbageParser( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ logger.error( "{} is unparsable",garbageString );
+ break;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private String uuidStringVerifier( final String garbageString ) {
+ int index = 1;
+ String stringToBeTruncated = garbageString;
+ while ( !UUIDUtils.isUUID( stringToBeTruncated ) ) {
+ if ( stringToBeTruncated.length() > 36 ) {
+ stringToBeTruncated = stringToBeTruncated.substring( index );
+ }
+ else {
+ return null;
+ }
+ }
+ return stringToBeTruncated;
+ }
+
+
+ private void deleteUniqueValue( final UUID applicationId, final String collectionName,
+ final String uniqueValueKey, final String uniqueValue, final UUID entityId )
+ throws Exception {
+ logger.warn( "Entity with id {} did not exist in app {} Deleting", entityId, applicationId );
+ UUID timestampUuid = newTimeUUID();
+ long timestamp = getTimestampInMicros( timestampUuid );
+ Keyspace ko = cass.getApplicationKeyspace( applicationId );
+ Mutator<ByteBuffer> mutator = createMutator( ko, be );
+
+ Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue );
+ addDeleteToMutator( mutator, ENTITY_UNIQUE, key, entityId, timestamp );
+ mutator.execute();
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/15e25dea/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
deleted file mode 100644
index 0d921d7..0000000
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ /dev/null
@@ -1,495 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-
-import java.io.File;
-import java.io.FileFilter;
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.junit.*;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.commons.lang.RandomStringUtils;
-
-import org.apache.usergrid.ServiceITSetup;
-import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.ServiceITSuite;
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.management.ManagementService;
-import org.apache.usergrid.management.OrganizationOwnerInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.entities.User;
-import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.mutation.MutationResult;
-import me.prettyprint.hector.api.mutation.Mutator;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
-import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.fail;
-
-
-/**
- * Created by ApigeeCorporation on 11/2/15.
- */
-public class UniqueIndexCleanupTest {
- static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
-
- int NUM_COLLECTIONS = 10;
- int NUM_ENTITIES = 50;
- int NUM_CONNECTIONS = 3;
-
- @ClassRule
- public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
-
-
- @org.junit.Test
- public void testBasicOperation() throws Exception {
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:9160"
- }, false );
-
- System.out.println( "completed" );
- }
-
- //this test is perfect for the other tool the userCollectionFix tool as this is what I believe they were seeing.
- @Ignore ("WRong test not made for unique index cleanup.")
- public void testRepairOfSingleEntityMissingColumnWrongTool() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,username,username,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
- CassandraService cass = setup.getCassSvc();
-
- List<HColumn<ByteBuffer, ByteBuffer>> cols =
- cass.getColumns( cass.getApplicationKeyspace( applicationInfo.getId() ), ENTITY_UNIQUE, key, null, null,
- 2, false );
-
- Set<UUID> results = new HashSet<UUID>( cols.size() );
-
- for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) {
- results.add( ue.fromByteBuffer( col.getName() ) );
- }
-
- UUID uuid = results.iterator().next();
-
- UUID timestampUuid = newTimeUUID();
- long timestamp = getTimestampInMicros( timestampUuid );
-
- //Keyspace ko = cass.getUsergridApplicationKeyspace();
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- key = key( applicationInfo.getId(), collectionName, "username", username );
- //addDeleteToMutator( m, ENTITY_UNIQUE, key, uuid, timestamp );
- addDeleteToMutator( m, ENTITY_UNIQUE, key, timestamp, uuid );
- m.execute();
-
- assertNull( entityManager.getAlias( applicationInfo.getId(), collectionName, username ) );
-
- assertNotNull(entityManager.get( entityToBeCorrupted.getUuid() ));
-
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
-
-
- //here you need to add a delete to the mutator then recheck it and see if the entity is the same as .
- Thread.sleep( 2000 );
- assertNull( entityManager.get( entityToBeCorrupted.getUuid() ) );
-
- //When you come back you also need to emulate the tests to delete what is out of the uuid without any corresponding data.
- //maybe it'll be easier to just do an insert into the EntityUnique row without doint it into any other row and
- //then verify the data like that. Then you don't have to do deletes out of other things.
-
- }
-
- //For this test you need to insert a dummy key with a dummy column that leads to nowhere
- //then run the unique index cleanup.
- //checks for bug when only column doesn't exist make sure to delete the row as well.
-
- //due to the read repair this is no longer a valid test of hte unique index cleanup
- @Ignore
- public void testRepairOfSingleEntity() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name
- addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
-
- m.execute();
-
- //verify that there is no corresponding entity with the uuid or alias provided
- //verify it returns null.
- //assertNull(entityManager.get( testEntityUUID ));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- //The Get above should have repaired the entity allowing it run
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- //fail();
- }catch(DuplicateUniquePropertyExistsException dup){
- fail();
- }
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
-
- //run the cleanup
-// UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
-// uniqueIndexCleanup.startTool( new String[] {
-// "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
-// }, false );
-
-
- entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- assertNotNull( entityToBeCorrupted );
- assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername());
-
- }
-
- //For this test you need to insert a dummy key with a dummy column that leads to nowhere
- //then run the unique index cleanup.
- //checks for bug when only column doesn't exist make sure to delete the row as well.
- // Due to the read repair this is no longer a valid test of unique index cleanup.
- @Test
- @Ignore
- public void testRepairOfMultipleEntities() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- int numberOfEntitiesToCreate = 1000;
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- String[] usernames = new String[numberOfEntitiesToCreate];
-
- int index = 0;
- while(index < numberOfEntitiesToCreate) {
-
- usernames[index]=username+index;
-
- Map<String, Object> userInfo = new HashMap<String, Object>();
- userInfo.put( "username", usernames[index] );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", usernames[index] );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name
- addInsertToMutator( m, ENTITY_UNIQUE, key, testEntityUUID, 0, createTimestamp() );
-
- m.execute();
- index++;
-
- //the below works but not needed for this test.
- //assertNull( entityManager.getAlias("user",username));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
- }
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
- }
-
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
-
- for(String user:usernames ) {
- Map<String, Object> userInfo = new HashMap<String, Object>();
- userInfo.put( "username", user);
- Entity entityToBeCorrupted = entityManager.create( collectionName, userInfo );
-
- assertNotNull( entityToBeCorrupted );
- assertEquals( user, ( ( User ) entityToBeCorrupted ).getUsername() );
- }
- }
-
- @Test
- public void testRepairOfOnlyOneOfTwoColumns() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- //create a new column
- Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
- addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
-
- m.execute();
-
- //verify that there is no corresponding entity with the uuid or alias provided
- //verify it returns null.
- assertNull(entityManager.get( testEntityUUID ));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
- }
- catch(Exception e){
- fail("shouldn't throw something else i think");
- }
-
-
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort()
- }, false );
-
- //verifies it works now.
- assertNotNull( entityManager
- .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
-
- }
-
- @Test
- public void testStringParsing(){
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
-
- //String garbageString = ")xƐ:�^Q��I�p\\�/2178c690-3a6f-11e4-aec6-48fa705cb26f:users:username:test";
-
- UUID uuid = UUIDUtils.newTimeUUID();
-
- String garbageString = "S2^? <-^Q��%\"�]^S:"+uuid+":users:username:2";
-
- String[] parsedString = garbageString.split( ":" );
-
- String[] cleanedString = uniqueIndexCleanup.garbageRowKeyParser( parsedString );
-
- assertEquals( uuid.toString(),cleanedString[0] );
- assertEquals( "users",cleanedString[1] );
- assertEquals( "username",cleanedString[2] );
- assertEquals( "2",cleanedString[3] );
- }
-
- //POinting at single values is broken now but not entirely used right now anyways.
- //@Ignore
- @Test
- public void testRepairOfOnlyOneOfTwoColumnsWhilePointingAtSingleValue() throws Exception{
- String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
- String orgName = "org_" + rand;
- String appName = "app_" +rand;
- String username = "username_" + rand;
- String adminUsername = "admin_"+rand;
- String email = username+"@derp.com";
- String password = username;
-
- String collectionName = "users";
-
-
- OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password );
-
- ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName );
-
- EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() );
-
- Map<String,Object> userInfo = new HashMap<String, Object>( );
- userInfo.put( "username",username );
-
- //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo );
-
- CassandraService cass = setup.getCassSvc();
-
- Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username );
-
- Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() );
- Mutator<ByteBuffer> m = createMutator( ko, be );
-
- //create a new column
- Entity validCoexistingEntity = entityManager.create( collectionName, userInfo );
-
- UUID testEntityUUID = UUIDUtils.newTimeUUID();
- //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value.
- addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp());
-
- m.execute();
-
- //verify that there is no corresponding entity with the uuid or alias provided
- //verify it returns null.
- assertNull(entityManager.get( testEntityUUID ));
-
- //verify that we cannot recreate the entity due to duplicate unique property exception
- Entity entityToBeCorrupted = null;
- try {
- entityToBeCorrupted = entityManager.create( collectionName, userInfo );
- fail();
- }catch(DuplicateUniquePropertyExistsException dup){
-
- }
- catch(Exception e) {
- fail( "shouldn't throw something else i think" );
- }
-
- //NEED TO FAIL MORE GRACEFULLY
- //run the cleanup
- UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
- uniqueIndexCleanup.startTool( new String[] {
- "-host", "localhost:"+ ServiceITSuite.cassandraResource.getRpcPort(),
- "-col",collectionName,
- "-app",applicationInfo.getId().toString(),
- "-property","username",
- "-value",username
- }, false );
-
- //verifies it works now.
- assertNotNull( entityManager
- .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) );
-
- }
-}
-
[13/39] usergrid git commit: Fixed test and removed options that
weren't being used.
Posted by sn...@apache.org.
Fixed test and removed options that weren't being used.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6d96a4c9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6d96a4c9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6d96a4c9
Branch: refs/heads/1.x
Commit: 6d96a4c90abae8768e9fbd41001a5cc66bbe8949
Parents: 6e8b2b6
Author: George Reyes <gr...@apache.org>
Authored: Wed Nov 4 16:37:30 2015 -0800
Committer: George Reyes <gr...@apache.org>
Committed: Wed Nov 4 16:37:30 2015 -0800
----------------------------------------------------------------------
.../usergrid/tools/UniqueIndexCleanup.java | 70 +-------------------
.../usergrid/tools/UniqueIndexCleanupTest.java | 2 -
2 files changed, 2 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6d96a4c9/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
index 9b1c202..39ac762 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java
@@ -18,79 +18,36 @@ package org.apache.usergrid.tools;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.cassandra.db.RowIteratorFactory;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.io.Charsets;
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.Identifier;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.EntityManagerImpl;
-import org.apache.usergrid.persistence.cassandra.Serializers;
-import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
-import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
-import org.apache.usergrid.persistence.cassandra.index.UUIDStartToBytes;
-import org.apache.usergrid.persistence.entities.Application;
-import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
-import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
-import org.apache.usergrid.persistence.query.ir.result.UUIDIndexSliceParser;
-import org.apache.usergrid.persistence.schema.CollectionInfo;
-
-import me.prettyprint.cassandra.service.KeyIterator;
+
import me.prettyprint.cassandra.service.RangeSlicesIterator;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
-import me.prettyprint.hector.api.beans.DynamicComposite;
import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.beans.OrderedRows;
import me.prettyprint.hector.api.beans.Row;
-import me.prettyprint.hector.api.beans.Rows;
import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
-import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.RangeSlicesQuery;
-import me.prettyprint.hector.api.query.SliceQuery;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static me.prettyprint.hector.api.factory.HFactory.createRangeSlicesQuery;
-import static me.prettyprint.hector.api.factory.HFactory.createSliceQuery;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_COLLECTIONS;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
-import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_ID_SETS;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_PROPERTIES;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.APPLICATIONS_CF;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import static org.apache.usergrid.persistence.cassandra.Serializers.be;
-import static org.apache.usergrid.persistence.cassandra.Serializers.dce;
-import static org.apache.usergrid.persistence.cassandra.Serializers.le;
-import static org.apache.usergrid.persistence.cassandra.Serializers.se;
import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
-import static org.apache.usergrid.utils.ConversionUtils.bytebuffer;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
@@ -112,16 +69,6 @@ public class UniqueIndexCleanup extends ToolBase {
private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class );
- /**
- *
- */
- private static final String APPLICATION_ARG = "app";
-
- /**
- *
- */
- private static final String COLLECTION_ARG = "col";
-
@Override
@SuppressWarnings( "static-access" )
@@ -136,19 +83,6 @@ public class UniqueIndexCleanup extends ToolBase {
options.addOption( hostOption );
-
-
- Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
- .withDescription( "application id or app name" ).create( APPLICATION_ARG );
-
-
- options.addOption( appOption );
-
- Option collectionOption = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
- .withDescription( "collection name" ).create( COLLECTION_ARG );
-
- options.addOption( collectionOption );
-
return options;
}
@@ -176,7 +110,7 @@ public class UniqueIndexCleanup extends ToolBase {
//not sure if I trust the lower two settings as it might iterfere with paging or set
// arbitrary limits and what I want to retrieve.
//That needs to be verified.
- .setKeys( null, null ).setRange( null, null, false, 100 );
+ .setKeys( null, null ).setRange( null, null, false, PAGE_SIZE );
RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6d96a4c9/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
index 8cb6a88..194793c 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/UniqueIndexCleanupTest.java
@@ -68,8 +68,6 @@ public class UniqueIndexCleanupTest {
UniqueIndexCleanup uniqueIndexCleanup = new UniqueIndexCleanup();
uniqueIndexCleanup.startTool( new String[]{
- "-app", "942712f0-7ce2-11e5-b81a-17ac5477fa5c",
- "-col", "users",
"-host", "localhost:9160"
}, false );