You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/10/28 19:23:38 UTC
usergrid git commit: Added the collection tool that needs to be
expanded
Repository: usergrid
Updated Branches:
refs/heads/userCollectionFix [created] a72da10f8
Added the collection tool that needs to be expanded
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a72da10f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a72da10f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a72da10f
Branch: refs/heads/userCollectionFix
Commit: a72da10f85bd01b47950e18b4ab2f47dfa4ecf23
Parents: afe0c51
Author: George Reyes <gr...@apache.org>
Authored: Wed Oct 28 11:23:37 2015 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Wed Oct 28 11:23:37 2015 -0700
----------------------------------------------------------------------
.../rest/applications/users/UsersResource.java | 3 +
...cateUniquePropertyExistsExceptionMapper.java | 6 +
.../usergrid/tools/CollectionUserFix.java | 255 +++++++++++++++++++
3 files changed, 264 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
index 6325d5a..408299a 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UsersResource.java
@@ -100,6 +100,9 @@ public class UsersResource extends ServiceResource {
@Path("{itemName}")
public AbstractContextResource addNameParameter( @Context UriInfo ui, @PathParam("itemName") PathSegment itemName )
throws Exception {
+ //TODO: read repair if entity ref doesn't match returned entity. Detect index corruption and delete reference if
+ //reference isn't correct.
+
logger.info( "ServiceResource.addNameParameter" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
index b910b58..0ea1567 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/DuplicateUniquePropertyExistsExceptionMapper.java
@@ -29,6 +29,12 @@ import static javax.ws.rs.core.Response.Status.BAD_REQUEST;
public class DuplicateUniquePropertyExistsExceptionMapper
extends AbstractExceptionMapper<DuplicateUniquePropertyExistsException> {
+ //when you get this exception then fire a repair task that will grab a thread from a thread pool and async run the repair task
+ // That task will then go through and double checkk all the entities and verify that their ref's and entities match up.
+
+
+ //npe, delete entity ref
+ //mismatched pointers, update the entity ref with whatever is in the collection. ( retrieve via query ) .
@Override
public Response toResponse( DuplicateUniquePropertyExistsException e ) {
return toResponse( BAD_REQUEST, e );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a72da10f/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
new file mode 100644
index 0000000..829487a
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/CollectionUserFix.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Identifier;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.Application;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+
+
+/**
+ * Fixes issues where admin data is in organizations but can't accessed or reset the password of. This usually returns
+ * the error "Could not find organization for email" because the user can't be found but when the org is queried the
+ * user shows up there. If you see that then run this tool.
+ */
+public class CollectionUserFix extends ExportingToolBase {
+ private static final int PAGE_SIZE = 100;
+
+ private static final Logger logger = LoggerFactory.getLogger( CollectionUserFix.class );
+
+ /**
+ *
+ */
+ private static final String ORGANIZATION_ARG = "org";
+
+ /**
+ *
+ */
+ private static final String APPLICATION_ARG = "app";
+
+ /**
+ *
+ */
+ private static final String QUERY_ARG = "query";
+
+ @Override
+ @SuppressWarnings( "static-access" )
+ public Options createOptions() {
+
+ Option hostOption =
+ OptionBuilder.withArgName( "host" ).hasArg().isRequired( true ).withDescription( "Cassandra host" )
+ .create( "host" );
+
+ Option orgOption = OptionBuilder.withArgName( ORGANIZATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "organization id" ).create( ORGANIZATION_ARG );
+ Option appOption = OptionBuilder.withArgName( APPLICATION_ARG ).hasArg().isRequired( false )
+ .withDescription( "application uuid" ).create( APPLICATION_ARG );
+ Option queryOption = OptionBuilder.withArgName( QUERY_ARG ).hasArg().isRequired( false )
+ .withDescription( "query" ).create( QUERY_ARG );
+
+ //add a query to it
+
+ Options options = new Options();
+ options.addOption( hostOption );
+ options.addOption( orgOption );
+ options.addOption( appOption );
+ options.addOption( queryOption );
+
+ return options;
+ }
+
+
+ /**
+ * psudeo code
+ * if org id is present then ignore the application id
+ * go through everysingle application/users entity. And check to see if it can be queried by username.
+ * else if app id is present
+ * go through the applications/users
+ *
+ *
+ * Take the list of applications to go through and go through them one by one. ( in the latter else case we will only
+ *
+ * @param line
+ * @throws Exception
+ */
+
+
+ @Override
+ public void runTool( CommandLine line ) throws Exception {
+
+
+ startSpring();
+
+ logger.info( "Starting crawl of all users" );
+ System.out.println( "Starting crawl of all users" );
+ Set<Application> applicationSet = new HashSet<Application>( );
+ EntityManager em = null;
+
+
+
+ if(line.getOptionValue( ORGANIZATION_ARG )==null||line.getOptionValue( ORGANIZATION_ARG ).isEmpty()) {
+ em = emf.getEntityManager( UUID.fromString( line.getOptionValue( APPLICATION_ARG ) ) );
+ applicationSet.add( em.getApplication() );
+ }
+ else{
+ BiMap applicationsForOrganization =
+ managementService
+ .getApplicationsForOrganization( UUID.fromString(line.getOptionValue( ORGANIZATION_ARG )));
+
+ applicationSet = applicationsForOrganization.keySet();
+ }
+ startCollectionFlow( em, applicationSet, line.getOptionValue( QUERY_ARG ) );
+
+
+ logger.info( "Repair complete" );
+ System.out.println("Repair Complete");
+ }
+
+
+ //make
+ private void startCollectionFlow(final EntityManager entityManager, final Set<Application> app, final String queryString )
+ throws Exception {// search for all orgs
+
+ Query query = new Query();
+ if(queryString!=null){
+ query = query.fromQL( queryString );
+ }
+ query.setLimit( PAGE_SIZE );
+ Results r = null;
+ EntityManager em = null;
+
+ for ( Application application : app ) {
+ //This will hold all of the applications users. This will be stored in memory to do a simple check to see if
+ //there are any duped usernames in the collection.
+ //Memory concerns means that
+ Multimap<String, UUID> usernames = HashMultimap.create();
+
+ //This means that we need to set it for each and every single application thus it gets set here instead of
+ //the method that calls us.
+ if(entityManager == null){
+ em = emf.getEntityManager( application.getUuid() );
+ }
+ else {
+ em = entityManager;
+ }
+
+ do {
+
+ //get all users in the management app and page for each set of a PAGE_SIZE
+ r = em.searchCollection( application, "users", query );
+ System.out.println("found "+r.size()+" number of entities");
+
+ for ( Entity entity : r.getEntities() ) {
+ //grab all usernames returned.
+ usernames.put( entity.getProperty( "username" ).toString().toLowerCase(), entity.getUuid() );
+ }
+
+ query.setCursor( r.getCursor() );
+
+ logger.info( "Searching next page" );
+ System.out.println("Searching next page");
+ }
+ while ( r != null && r.size() == PAGE_SIZE );
+
+ System.out.println("Starting username crawl of "+usernames.size()+" number of usernames");
+ //do a get on a specific username, if it shows up more than once then remove it
+ for ( String username : usernames.keySet() ) {
+ Collection<UUID> ids = usernames.get( username );
+
+ if ( ids.size() > 1 ) {
+ logger.info( "Found multiple users with the username {}", username );
+ System.out.println( "Found multiple users with the username: " + username );
+ }
+
+ //UserInfo targetUser = managementService.getAdminUserByEmail( email );
+ Identifier identifier = new Identifier();
+ EntityRef targetUser = em.getUserByIdentifier( identifier.fromName( username ) );
+
+
+ if ( targetUser == null ) {
+ //This means that the username isn't properly associated with targetUser
+ List<UUID> tempIds = new ArrayList<UUID>( ids );
+ //Collections.sort( tempIds );
+
+ UUID toLoad = tempIds.get( 0 );
+
+ logger.warn( "Could not load target user by username {}, loading by UUID {} instead", username, toLoad );
+ System.out.println( "Could not load the target user by username: " + username
+ + ". Loading by the following uuid instead: " + toLoad.toString() );
+
+ User targetUserEntity = null;
+ try {
+ targetUserEntity = em.get( toLoad, User.class );
+ }catch(Exception e){
+ System.out.println("The follow uuid has no data in this cassandra node: "+toLoad.toString());
+ throw e;
+ }
+
+
+ try {
+ if ( targetUserEntity != null&& targetUserEntity.getUuid().equals( toLoad )) {
+ System.out.println("Updating uuid: "+targetUserEntity.getUuid().toString());
+ em.update( targetUserEntity );
+ }
+ }
+ catch ( DuplicateUniquePropertyExistsException dup ) {
+ System.out.println( "Found duplicate unique property: " + dup.getPropertyName() + ". "
+ + "Duplicate property is: "
+ + dup.getPropertyValue() );
+ //if there are duplicate unique properties then
+ if ( dup.getPropertyName().equals( "username" ) ) {
+ System.out.println("can I replace this with a different value since these are duplicated in the code base");
+ //targetUserEntity.setUsername( targetUserEntity.getUsername() );
+ }
+ //else throw dup;
+ }
+ catch (Exception e){
+ System.out.println("There was an issue with updating: "+e.getMessage());
+ }
+ }
+ }
+ }
+ }
+}