You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2016/01/22 15:36:47 UTC
[1/3] usergrid git commit: "port" tools from master
Repository: usergrid
Updated Branches:
refs/heads/release 62ad5a840 -> 46fb92652
"port" tools from master
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e5ebb375
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e5ebb375
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e5ebb375
Branch: refs/heads/release
Commit: e5ebb375386f34b2ad8883c28701500b59208a81
Parents: 4e048f3
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jan 22 09:31:06 2016 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jan 22 09:31:06 2016 -0500
----------------------------------------------------------------------
.../usergrid/management/ManagementService.java | 4 +-
.../cassandra/ManagementServiceImpl.java | 10 +-
.../usergrid/tools/DryRunUserOrgManager.java | 62 +++
.../usergrid/tools/DuplicateAdminRepair.java | 293 +++++++++++++
.../usergrid/tools/DuplicateOrgRepair.java | 421 +++++++++++++++++++
.../org/apache/usergrid/tools/ExportAdmins.java | 38 +-
.../apache/usergrid/tools/UserOrgInterface.java | 173 ++++++++
.../apache/usergrid/tools/UserOrgManager.java | 389 +++++++++++++++++
.../apache/usergrid/tools/ExportAppTest.java | 4 -
9 files changed, 1372 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5ebb375/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index cf2924b..0851838 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -229,7 +229,9 @@ public interface ManagementService {
String objectType, String objectName, String title, String content )
throws Exception;
- public void removeAdminUserFromOrganization( UUID userId, UUID organizationId ) throws Exception;
+ public void removeAdminUserFromOrganization(UUID userId, UUID organizationId ) throws Exception;
+
+ public void removeAdminUserFromOrganization(UUID userId, UUID organizationId, boolean force ) throws Exception;
public void removeOrganizationApplication( UUID organizationId, UUID applicationId ) throws Exception;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5ebb375/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index f252705..fe8152d 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -1585,7 +1585,13 @@ public class ManagementServiceImpl implements ManagementService {
@Override
- public void removeAdminUserFromOrganization( UUID userId, UUID organizationId ) throws Exception {
+ public void removeAdminUserFromOrganization(UUID userId, UUID organizationId ) throws Exception {
+ removeAdminUserFromOrganization( userId, organizationId, false );
+ }
+
+
+ @Override
+ public void removeAdminUserFromOrganization(UUID userId, UUID organizationId, boolean force) throws Exception {
if ( ( userId == null ) || ( organizationId == null ) ) {
return;
@@ -1594,7 +1600,7 @@ public class ManagementServiceImpl implements ManagementService {
EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
try {
- if ( em.getCollection( new SimpleEntityRef( Group.ENTITY_TYPE, organizationId ), "users", null, 2,
+ if ( !force && em.getCollection( new SimpleEntityRef( Group.ENTITY_TYPE, organizationId ), "users", null, 2,
Level.IDS, false ).size() <= 1 ) {
throw new Exception();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5ebb375/stack/tools/src/main/java/org/apache/usergrid/tools/DryRunUserOrgManager.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DryRunUserOrgManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DryRunUserOrgManager.java
new file mode 100644
index 0000000..966b18c
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/DryRunUserOrgManager.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+
+import java.util.UUID;
+
+class DryRunUserOrgManager extends UserOrgManager {
+
+ public DryRunUserOrgManager(EntityManagerFactory emf, ManagementService managementService) {
+ super( emf, managementService );
+ }
+
+ @Override
+ public void removeUserFromOrg(OrgUser user, Org org) throws Exception {
+ }
+
+ @Override
+ public void addUserToOrg(OrgUser user, Org org) throws Exception {
+ }
+
+ @Override
+ public void addAppToOrg(UUID appId, Org org) throws Exception {
+ }
+
+ @Override
+ public void removeOrgUser(OrgUser orgUser) {
+ }
+
+ @Override
+ public void updateOrgUser(OrgUser targetUserEntity) {
+ }
+
+ @Override
+ public void setOrgUserName(OrgUser other, String newUserName) {
+
+ }
+
+ @Override
+ public void removeAppFromOrg(UUID appId, Org org) throws Exception {
+ }
+
+ @Override
+ public void removeOrg(Org keeper, Org duplicate) throws Exception {
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5ebb375/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateAdminRepair.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateAdminRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateAdminRepair.java
new file mode 100644
index 0000000..172199c
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateAdminRepair.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import rx.functions.Action1;
+
+import java.util.*;
+
+import static org.apache.usergrid.tools.UserOrgInterface.Org;
+import static org.apache.usergrid.tools.UserOrgInterface.OrgUser;
+
+
+/**
+ * Find duplicate admin users, delete the one that is not indexed.
+ */
+public class DuplicateAdminRepair extends ToolBase {
+
+ UserOrgInterface manager = null;
+
+ static final String THREADS_ARG_NAME = "threads";
+
+ int threadCount = 5;
+
+ static final String DRYRUN_ARG_NAME = "dryrun";
+
+ boolean dryRun = false;
+
+ Multimap<String, OrgUser> emails = HashMultimap.create();
+
+ Multimap<String, OrgUser> usernames = HashMultimap.create();
+
+ boolean testing = false;
+
+
+ DuplicateAdminRepair() {
+ super();
+ }
+
+ DuplicateAdminRepair(EntityManagerFactory emf, ManagementService managementService ) {
+ this();
+ this.emf = emf;
+ this.managementService = managementService;
+ }
+
+
+ @Override
+ @SuppressWarnings("static-access")
+ public Options createOptions() {
+
+ Options options = super.createOptions();
+
+ Option dryRunOption = OptionBuilder.hasArg()
+ .withType(Boolean.TRUE)
+ .withDescription( "-" + DRYRUN_ARG_NAME + " true to print what tool would do and do not alter data.")
+ .create( DRYRUN_ARG_NAME );
+ options.addOption( dryRunOption );
+
+ Option writeThreadsOption = OptionBuilder.hasArg()
+ .withType(0)
+ .withDescription( "Write Threads -" + THREADS_ARG_NAME )
+ .create(THREADS_ARG_NAME);
+ options.addOption( writeThreadsOption );
+
+ return options;
+ }
+
+
+ public UserOrgManager createNewRepairManager() {
+ return new UserOrgManager( emf, managementService );
+ }
+
+
+ @Override
+ public void runTool(CommandLine line) throws Exception {
+
+ startSpring();
+ setVerbose( line );
+
+ if (StringUtils.isNotEmpty( line.getOptionValue( THREADS_ARG_NAME ) )) {
+ try {
+ threadCount = Integer.parseInt( line.getOptionValue( THREADS_ARG_NAME ) );
+ } catch (NumberFormatException nfe) {
+ logger.error( "-" + THREADS_ARG_NAME + " must be specified as an integer. Aborting..." );
+ return;
+ }
+ }
+
+ if ( StringUtils.isNotEmpty( line.getOptionValue( DRYRUN_ARG_NAME ) )) {
+ dryRun = Boolean.parseBoolean( line.getOptionValue( DRYRUN_ARG_NAME ));
+ }
+
+ if ( manager == null ) { // we use a special manager when mockTesting
+ if (dryRun) {
+ manager = new DryRunUserOrgManager( emf, managementService );
+ } else {
+ manager = new UserOrgManager( emf, managementService );
+ }
+ }
+
+ logger.info( "DuplicateAdminUserRepair tool starting up... manager: " + manager.getClass().getSimpleName() );
+
+ // build multi-map of users by email and users by name
+
+ manager.getUsers().doOnNext( new Action1<OrgUser>() {
+ @Override
+ public void call( OrgUser user ) {
+
+ if (user.getUsername() == null) {
+ logger.warn( "User {} has no username", user.getId() );
+ return;
+ }
+ if (user.getEmail() == null) {
+ logger.warn( "User {} has no email", user.getId() );
+ return;
+ }
+ emails.put( user.getEmail().toLowerCase(), user );
+ usernames.put( user.getUsername().toLowerCase(), user );
+
+ }
+ } ).toBlocking().lastOrDefault( null );
+
+ for ( String username : usernames.keySet() ) {
+ Collection<OrgUser> users = usernames.get( username );
+
+ if ( users.size() > 1 ) {
+ logger.info( "Found multiple users with the username {}", username );
+
+ // force the username to be reset to the user's email
+ resolveUsernameConflicts( username, users );
+ }
+ }
+
+ for ( String email : emails.keySet() ) {
+
+ // go through a set of users with duplicate emails
+ Collection<OrgUser> usersWithDupEmails = emails.get( email );
+
+ if ( usersWithDupEmails.size() > 1 ) {
+
+ // get the user that is in the users-by-email index, it's the keeper
+ OrgUser indexedUser = manager.lookupOrgUserByEmail( email );
+
+ if ( indexedUser == null ) {
+
+ // no user is indexed with that email, pick oldest as keeper
+ List<OrgUser> tempUsers = new ArrayList<OrgUser>( usersWithDupEmails );
+ Collections.sort( tempUsers );
+ OrgUser oldestOfLot = tempUsers.get( 0 );
+
+ logger.warn( "Could not load target user by email {}, loading by UUID {} instead", email, oldestOfLot );
+ indexedUser = oldestOfLot;
+ }
+
+ logger.warn( "Found multiple admins with the email {}. Retaining uuid {}", email, indexedUser.getId() );
+
+ for ( OrgUser orgUser : usersWithDupEmails ) {
+ if ( !orgUser.getId().equals( indexedUser.getId() )) {
+ mergeAdmins( orgUser, indexedUser );
+ }
+ }
+
+ // force the index update after all other admins have been merged
+ if ( dryRun ) {
+ logger.info("Would force re-index of 'keeper' user {}:{}",
+ indexedUser.getUsername(), indexedUser.getId());
+ } else {
+ logger.info( "Forcing re-index of admin with email {} and id {}", email, indexedUser.getId());
+ manager.updateOrgUser( indexedUser );
+ }
+ }
+ }
+
+ logger.info( "Repair complete" );
+ }
+
+
+ /**
+ * When our usernames are equal, we need to check if our emails are equal. If they're not, we need to change the one
+ * that DOES NOT get returned on a lookup by username
+ */
+ private void resolveUsernameConflicts( String userName, Collection<OrgUser> users ) throws Exception {
+
+ // lookup the admin id
+ OrgUser existing = manager.lookupOrgUserByUsername( userName );
+
+ if ( existing == null ) {
+ logger.warn( "Could not determine an admin for colliding username '{}'. Skipping", userName );
+ return;
+ }
+
+ users.remove( existing );
+
+ boolean collision = false;
+
+ for ( OrgUser other : users ) {
+
+ // same username and email, these will be merged later in the process,
+ // skip it
+ if ( other != null && other.getEmail() != null && other.getEmail().equals( existing.getEmail() ) ) {
+ logger.info(
+ "Users with the same username '{}' have the same email '{}'. This will be resolved later in "
+ + "the process, skipping", userName, existing.getEmail() );
+ continue;
+ }
+
+ // if we get here, the emails do not match, but the usernames do. Force
+ // both usernames to emails
+ collision = true;
+
+ setUserName(other, other.getEmail() );
+ }
+
+ if ( collision ) {
+ setUserName(existing, existing.getEmail() );
+ }
+ }
+
+
+ /** Set the username to the one provided, if we can't due to duplicate property issues, we fall back to user+uuid */
+ private void setUserName( OrgUser other, String newUserName ) throws Exception {
+
+ if ( dryRun ) {
+ logger.info("Would rename user {}:{} to {}", new Object[] {
+ other.getUsername(), other.getId(), newUserName });
+ } else {
+ logger.info( "Setting username to {} for user {}:{}", new Object[] {
+ newUserName, other.getUsername(), other.getId() } );
+
+ manager.setOrgUserName( other, newUserName );
+ }
+ }
+
+
+ /** Merge the source admin to the target admin by copying oranizations. Then deletes the source admin */
+ private void mergeAdmins( OrgUser sourceUser, OrgUser targetUser ) throws Exception {
+
+ logger.info("---> Merging user {}:{} into {}:{}", new Object[] {
+ sourceUser.getEmail(), sourceUser.getId(), targetUser.getEmail(), targetUser.getId()
+ } );
+
+ Set<Org> sourceOrgs = manager.getUsersOrgs( sourceUser );
+
+ for ( Org org : sourceOrgs ) {
+
+ if ( dryRun ) {
+ logger.info( "Would add organization {}:{} to admin with email {} and id {}",
+ new Object[] { org.getName(), org.getId(), targetUser.getEmail(), targetUser.getId() } );
+
+ } else {
+ logger.info( "Adding organization {}:{} to admin with email {} and id {}",
+ new Object[] { org.getName(), org.getId(), targetUser.getEmail(), targetUser.getId() } );
+
+ // add targetUser to sourceUser's org
+ manager.addUserToOrg( targetUser, org );
+ }
+ }
+
+ if ( dryRun ) {
+ logger.info( "Would remove admin with email {} and id {} from system",
+ sourceUser.getEmail(), sourceUser.getId() );
+
+ } else {
+ logger.info( "Removing admin with email {} and id {} from system",
+ sourceUser.getEmail(), sourceUser.getId() );
+
+ manager.removeOrgUser( sourceUser );
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5ebb375/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
new file mode 100644
index 0000000..0f93ed4
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
@@ -0,0 +1,421 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.persistence.*;
+import rx.Scheduler;
+import rx.functions.Action1;
+import rx.schedulers.Schedulers;
+
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import static org.apache.usergrid.tools.UserOrgInterface.Org;
+import static org.apache.usergrid.tools.UserOrgInterface.OrgUser;
+
+/**
+ * Find duplicate orgs, delete all but oldest of each and assign users to it.
+ */
+public class DuplicateOrgRepair extends ToolBase {
+
+ UserOrgInterface manager = null;
+
+ Map<String, Set<Org>> orgsByName = new HashMap<String, Set<Org>>();
+
+ Map<UUID, Org> orgsById = new HashMap<UUID, Org>();
+
+ Map<OrgUser, Set<Org>> orgsByUser = new HashMap<OrgUser, Set<Org>>();
+
+ Map<String, Set<Org>> duplicatesByName = new HashMap<String, Set<Org>>();
+
+ static final String THREADS_ARG_NAME = "threads";
+
+ int threadCount = 5;
+
+ static final String DRYRUN_ARG_NAME = "dryrun";
+
+ boolean dryRun = false;
+
+ static final String NO_AUGMENT_ARG_NAME = "noaugment";
+
+ boolean noAugment = false;
+
+ static final String ORG1_ID = "org1";
+
+ static final String ORG2_ID = "org2";
+
+ boolean testing = false;
+
+
+ DuplicateOrgRepair() {
+ super();
+ }
+
+ DuplicateOrgRepair( EntityManagerFactory emf, ManagementService managementService ) {
+ this();
+ this.emf = emf;
+ this.managementService = managementService;
+ }
+
+ @Override
+ @SuppressWarnings("static-access")
+ public Options createOptions() {
+
+ Options options = super.createOptions();
+
+ Option dryRunOption = OptionBuilder.hasArg()
+ .withType(Boolean.TRUE)
+ .withDescription( "-" + DRYRUN_ARG_NAME + " true to print what tool would do and do not alter data.")
+ .create( DRYRUN_ARG_NAME );
+ options.addOption( dryRunOption );
+
+ Option noAugmentRunOption = OptionBuilder.hasArg()
+ .withType(Boolean.TRUE)
+ .withDescription( "-" + NO_AUGMENT_ARG_NAME + " true to skip the augment step.")
+ .create( NO_AUGMENT_ARG_NAME );
+ options.addOption( noAugmentRunOption );
+
+
+ Option writeThreadsOption = OptionBuilder.hasArg()
+ .withType(0)
+ .withDescription( "Write Threads -" + THREADS_ARG_NAME )
+ .create(THREADS_ARG_NAME);
+ options.addOption( writeThreadsOption );
+
+ Option org1Option = OptionBuilder.hasArg()
+ .withType(0)
+ .withDescription( "Duplicate org #1 id -" + ORG1_ID)
+ .create(ORG1_ID);
+ options.addOption( org1Option );
+
+ Option org2Option = OptionBuilder.hasArg()
+ .withType(0)
+ .withDescription( "Duplicate org #2 id -" + ORG2_ID)
+ .create(ORG2_ID);
+ options.addOption( org2Option );
+
+ return options;
+ }
+
+ @Override
+ public void runTool(CommandLine line) throws Exception {
+
+ startSpring();
+ setVerbose( line );
+
+ UUID org1uuid = null;
+ UUID org2uuid = null;
+
+ String org1string = line.getOptionValue( ORG1_ID );
+ String org2string = line.getOptionValue( ORG2_ID );
+
+ if ( org1string != null && org2string == null ) {
+ logger.error("- if {} is specified you must also specify {} and vice-versa", ORG1_ID, ORG2_ID);
+ return;
+
+ } else if ( org2string != null && org1string == null ) {
+ logger.error("- if {} is specified you must also specify {} and vice-versa", ORG2_ID, ORG1_ID);
+ return;
+
+ } else if ( org1string != null && org2string != null ) {
+
+ try {
+ org1uuid = UUID.fromString( org1string );
+ org2uuid = UUID.fromString( org2string );
+ } catch (Exception e) {
+ logger.error("{} and {} must be specified as UUIDs", ORG1_ID, ORG2_ID);
+ return;
+ }
+ }
+
+ if (StringUtils.isNotEmpty( line.getOptionValue( THREADS_ARG_NAME ) )) {
+ try {
+ threadCount = Integer.parseInt( line.getOptionValue( THREADS_ARG_NAME ) );
+ } catch (NumberFormatException nfe) {
+ logger.error( "-" + THREADS_ARG_NAME + " must be specified as an integer. Aborting..." );
+ return;
+ }
+ }
+
+ if ( StringUtils.isNotEmpty( line.getOptionValue( DRYRUN_ARG_NAME ) )) {
+ dryRun = Boolean.parseBoolean( line.getOptionValue( DRYRUN_ARG_NAME ));
+ }
+
+ if ( StringUtils.isNotEmpty( line.getOptionValue( NO_AUGMENT_ARG_NAME ) )) {
+ noAugment = Boolean.parseBoolean( line.getOptionValue( NO_AUGMENT_ARG_NAME ));
+ }
+
+ if ( manager == null ) { // we use a special manager when mockTesting
+ if (dryRun) {
+ manager = new DryRunUserOrgManager( emf, managementService );
+ } else {
+ manager = new UserOrgManager( emf, managementService );
+ }
+ }
+
+ logger.info( "DuplicateOrgRepair tool starting up... manager: " + manager.getClass().getSimpleName() );
+
+ if ( org1uuid != null && org2uuid != null ) {
+
+ Org org1 = manager.getOrg( org1uuid );
+ Org org2 = manager.getOrg( org2uuid );
+
+ if ( org1.getName().equalsIgnoreCase( org2.getName() )) {
+ buildOrgMaps( org1, org2 );
+ } else {
+ logger.error("org1 and org2 do not have same duplicate name");
+ return;
+ }
+
+ } else {
+ buildOrgMaps();
+ }
+
+ if ( noAugment == false ) {
+ augmentUserOrgsMap();
+ }
+
+ manager.logDuplicates( duplicatesByName );
+
+ mergeDuplicateOrgs();
+
+ removeDuplicateOrgs();
+
+ logger.info( "DuplicateOrgRepair work is done!");
+ }
+
+
+ public UserOrgManager createNewRepairManager() {
+ return new UserOrgManager( emf, managementService );
+ }
+
+
+ private void buildOrgMaps(Org org1, Org org2) {
+
+ Set<Org> orgs = new HashSet<Org>();
+ orgs.add( org1 );
+ orgs.add( org2 );
+ orgsByName.put( org1.getName().toLowerCase(), orgs );
+ duplicatesByName.put( org1.getName().toLowerCase(), orgs );
+
+ orgsById.put( org1.getId(), org1 );
+ orgsById.put( org2.getId(), org2 );
+
+ for ( Org org : orgs ) {
+ try {
+ Set<OrgUser> orgUsers = manager.getOrgUsers( org );
+ for (OrgUser user : orgUsers) {
+ Set<Org> usersOrgs = orgsByUser.get( user );
+ if (usersOrgs == null) {
+ usersOrgs = new HashSet<Org>();
+ orgsByUser.put( user, usersOrgs );
+ }
+ usersOrgs.add( org );
+ }
+ } catch (Exception e) {
+ logger.error( "Error getting users for org {}:{}", org.getName(), org.getId() );
+ logger.error( "Stack trace is: ", e );
+ }
+ }
+
+ }
+
+
+ /**
+ * build map of orgs by name, orgs by id, orgs by user and duplicate orgs by name
+ */
+ private void buildOrgMaps() throws Exception {
+
+ manager.getOrgs().doOnNext( new Action1<Org>() {
+ @Override
+ public void call(Org org) {
+
+ // orgs by name and duplicate orgs by name maps
+
+ Set<Org> orgs = orgsByName.get( org.getName().toLowerCase() );
+ if (orgs == null) {
+ orgs = new HashSet<Org>();
+ orgsByName.put( org.getName().toLowerCase(), orgs );
+ } else {
+ duplicatesByName.put( org.getName().toLowerCase(), orgs );
+ }
+ orgs.add( org );
+
+ orgsById.put( org.getId(), org );
+
+ // orgs by user map, created via org -> user connections
+
+ try {
+ Set<OrgUser> orgUsers = manager.getOrgUsers( org );
+ for ( OrgUser user : orgUsers ) {
+ Set<Org> usersOrgs = orgsByUser.get( user );
+ if (usersOrgs == null) {
+ usersOrgs = new HashSet<Org>();
+ orgsByUser.put( user, usersOrgs );
+ }
+ usersOrgs.add( org );
+ }
+ } catch (Exception e) {
+ logger.error("Error getting users for org {}:{}", org.getName(), org.getId());
+ logger.error("Stack trace is: ", e);
+ }
+
+ }
+
+ } ).toBlocking().lastOrDefault( null );
+
+ logger.info( "DuplicateOrgRepair tool built org maps");
+ }
+
+
+ /**
+ * augment user orgs map via user -> org connections
+ */
+ private void augmentUserOrgsMap() throws Exception {
+
+ ExecutorService writeThreadPoolExecutor = Executors.newFixedThreadPool( threadCount );
+ Scheduler scheduler = Schedulers.from( writeThreadPoolExecutor );
+
+ manager.getUsers().doOnNext( new Action1<OrgUser>() {
+ @Override
+ public void call(OrgUser user) {
+ try {
+ Set<Org> connectedToOrgs = manager.getUsersOrgs(user);
+ Set<Org> usersOrgs = orgsByUser.get(user);
+ if ( usersOrgs == null ) {
+ usersOrgs = new HashSet<Org>();
+ }
+ for ( Org org : connectedToOrgs ) {
+ if (!usersOrgs.contains(org)) {
+ usersOrgs.add(org);
+ }
+ }
+
+ } catch (Exception e) {
+ logger.error("Error getting orgs for user {}:{}", user.getUsername(), user.getId());
+ logger.error("Stack trace is: ", e);
+ }
+ }
+ } ).subscribeOn( scheduler ).toBlocking().lastOrDefault( null );
+
+ logger.info( "DuplicateOrgRepair augmented user orgs map");
+ }
+
+
+ /**
+ * For each duplicate name, pick best org and merge apps and users into it
+ */
+ private void mergeDuplicateOrgs() throws Exception {
+
+ for ( String dupName : duplicatesByName.keySet() ) {
+ Set<Org> duplicateOrgs = duplicatesByName.get(dupName);
+ Org bestOrg = manager.selectBest( duplicateOrgs );
+
+ for ( Org org : duplicateOrgs ) {
+
+ if ( !org.equals( bestOrg )) {
+
+ Set<OrgUser> orgUsers = new HashSet<OrgUser>( manager.getOrgUsers( org ) );
+
+ for (OrgUser user : orgUsers) {
+ if (dryRun) {
+ Object[] args = new Object[]{
+ user.getUsername(), user.getId(), bestOrg.getName(), bestOrg.getId()};
+ logger.info( "Would add user {}:{} to org {}:{}", args );
+ args = new Object[]{
+ user.getUsername(), user.getId(), org.getName(), org.getId()};
+ logger.info( "Would remove user {}:{} org {}:{}", args );
+ } else {
+ try {
+ manager.addUserToOrg( user, bestOrg );
+ } catch ( Exception e ) {
+ Object[] args = new Object[]{
+ user.getUsername(), user.getId(), bestOrg.getName(), bestOrg.getId()};
+ logger.error( "Error adding user {}:{} to org {}:{}", args );
+ }
+ try {
+ manager.removeUserFromOrg( user, org );
+ } catch ( Exception e ) {
+ Object[] args = new Object[]{
+ user.getUsername(), user.getId(), org.getName(), org.getId()};
+ logger.info( "Error removing user {}:{} org {}:{}", args );
+ }
+ }
+ }
+
+ Set<UUID> orgApps = new HashSet<UUID>( manager.getOrgApps( org ) );
+
+ for (UUID appId : orgApps) {
+ if (dryRun) {
+ Object[] args = new Object[]{ appId, bestOrg.getName(), bestOrg.getId()};
+ logger.info( "Would add app {} to org {}:{}", args );
+ args = new Object[]{ appId, org.getName(), org.getId()};
+ logger.info( "Would remove app {} org {}:{}", args );
+ } else {
+ try {
+ manager.addAppToOrg( appId, bestOrg );
+ } catch ( Exception e ) {
+ Object[] args = new Object[]{ appId, bestOrg.getName(), bestOrg.getId()};
+ logger.error( "Error adding app {} to org {}:{}", args );
+ }
+ try {
+ manager.removeAppFromOrg( appId, org );
+ } catch (Exception e ) {
+ Object[] args = new Object[]{ appId, org.getName(), org.getId()};
+ logger.info( "Error removing app {} org {}:{}", args );
+ }
+ }
+ }
+
+ }
+ }
+ }
+
+ logger.info( "DuplicateOrgRepair merged duplicate orgs");
+ }
+
+
+ /**
+ * remove/rename duplicate orgs so they no longer impact operation of system
+ */
+ private void removeDuplicateOrgs() throws Exception {
+ for ( String dupName : duplicatesByName.keySet() ) {
+ Set<Org> orgs = duplicatesByName.get( dupName );
+ Org best = manager.selectBest( orgs );
+ for ( Org candidate : orgs ) {
+ if ( !candidate.equals(best) ) {
+ if ( dryRun ) {
+ logger.info("Would rename/remove org {}:{}",
+ new Object[] { candidate.getName(), candidate.getId() });
+ } else {
+ manager.removeOrg( best, candidate );
+ }
+ }
+ }
+ }
+
+ logger.info( "DuplicateOrgRepair renamed/removed duplicate orgs");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5ebb375/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index 479d740..070a499 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@ -16,6 +16,7 @@
*/
package org.apache.usergrid.tools;
+
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import org.apache.commons.cli.CommandLine;
@@ -28,6 +29,7 @@ import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.utils.StringUtils;
import org.codehaus.jackson.JsonGenerator;
import org.slf4j.Logger;
@@ -38,6 +40,7 @@ import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
+
/**
* Export Admin Users and metadata including organizations and passwords.
*
@@ -64,11 +67,12 @@ public class ExportAdmins extends ExportingToolBase {
public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";
// map admin user UUID to list of organizations to which user belongs
- private Map<UUID, List<Org>> userToOrgsMap = new HashMap<UUID, List<Org>>(50000);
+ private Map<UUID, List<Org>> userToOrgsMap = new HashMap<UUID, List<Org>>(100000);
- private Map<String, UUID> orgNameToUUID = new HashMap<String, UUID>(50000);
+ private Map<String, UUID> orgNameToUUID = new HashMap<String, UUID>(100000);
- private Set<UUID> orgsWritten = new HashSet<UUID>(50000);
+ private Set<UUID> orgsWritten = new HashSet<UUID>(100000);
+ private Set<String> orgsNamesWritten = new HashSet<String>(100000);
private Set<UUID> duplicateOrgs = new HashSet<UUID>();
@@ -146,10 +150,12 @@ public class ExportAdmins extends ExportingToolBase {
// start read queue workers
BlockingQueue<UUID> readQueue = new LinkedBlockingQueue<UUID>();
+ List<AdminUserReader> readers = new ArrayList<AdminUserReader>();
for (int i = 0; i < readThreadCount; i++) {
AdminUserReader worker = new AdminUserReader( readQueue, writeQueue );
Thread readerThread = new Thread( worker, "AdminUserReader-" + i );
readerThread.start();
+ readers.add( worker );
}
logger.debug( readThreadCount + " read worker threads started" );
@@ -191,7 +197,7 @@ public class ExportAdmins extends ExportingToolBase {
Options options = super.createOptions();
Option readThreads = OptionBuilder
- .hasArg().withType(0).withDescription("Read Threads -" + READ_THREAD_COUNT).create(READ_THREAD_COUNT);
+ .hasArg().withType(0).withDescription("Read Threads -" + READ_THREAD_COUNT).create(READ_THREAD_COUNT);
options.addOption( readThreads );
return options;
@@ -205,7 +211,7 @@ public class ExportAdmins extends ExportingToolBase {
logger.info( "Building org map" );
- ExecutorService execService = Executors.newFixedThreadPool( readThreadCount );
+ ExecutorService execService = Executors.newFixedThreadPool( this.readThreadCount );
EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
String queryString = "select *";
@@ -332,8 +338,8 @@ public class ExportAdmins extends ExportingToolBase {
String actionTaken = "Processed";
if (ignoreInvalidUsers && (task.orgNamesByUuid.isEmpty()
- || task.dictionariesByName.isEmpty()
- || task.dictionariesByName.get( "credentials" ).isEmpty())) {
+ || task.dictionariesByName.isEmpty()
+ || task.dictionariesByName.get( "credentials" ).isEmpty())) {
actionTaken = "Ignored";
@@ -342,9 +348,9 @@ public class ExportAdmins extends ExportingToolBase {
}
Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty() ?
- 0 : task.dictionariesByName.get( "credentials" ));
+ 0 : task.dictionariesByName.get( "credentials" ));
- logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}",
+ logger.warn( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}",
new Object[]{
actionTaken,
task.adminUser.getProperty( "username" ),
@@ -370,7 +376,7 @@ public class ExportAdmins extends ExportingToolBase {
Set<String> dictionaries = em.getDictionaries( entity );
if ( dictionaries.isEmpty() ) {
- logger.error("User {}:{} has no dictionaries", task.adminUser.getName(), task.adminUser.getUuid() );
+ logger.warn("User {}:{} has no dictionaries", task.adminUser.getName(), task.adminUser.getUuid() );
return;
}
@@ -385,7 +391,7 @@ public class ExportAdmins extends ExportingToolBase {
task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() );
- List<Org> orgs = userToOrgsMap.get( task.adminUser.getUuid() );
+ List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {
@@ -423,12 +429,12 @@ public class ExportAdmins extends ExportingToolBase {
// write one JSON file for management application users
JsonGenerator usersFile =
- getJsonGenerator( createOutputFile( ADMIN_USERS_PREFIX, em.getApplication().getName() ) );
+ getJsonGenerator( createOutputFile( ADMIN_USERS_PREFIX, em.getApplication().getName() ) );
usersFile.writeStartArray();
// write one JSON file for metadata: collections, connections and dictionaries of those users
JsonGenerator metadataFile =
- getJsonGenerator( createOutputFile( ADMIN_USER_METADATA_PREFIX, em.getApplication().getName() ) );
+ getJsonGenerator( createOutputFile( ADMIN_USER_METADATA_PREFIX, em.getApplication().getName() ) );
metadataFile.writeStartObject();
while ( true ) {
@@ -470,7 +476,8 @@ public class ExportAdmins extends ExportingToolBase {
usersFile.writeEndArray();
usersFile.close();
- logger.info( "Exported TOTAL {} admin users and {} organizations", userCount.get(), orgsWritten.size() );
+ logger.info( "Exported TOTAL {} admin users and {} organizations, org names = {}",
+ new Object[] { userCount.get(), orgsWritten.size(), orgsNamesWritten.size() } );
}
@@ -523,7 +530,9 @@ public class ExportAdmins extends ExportingToolBase {
jg.writeEndObject();
synchronized (orgsWritten) {
+ logger.info("Exported org {}:{}", uuid, orgs.get(uuid));
orgsWritten.add( uuid );
+ orgsNamesWritten.add( orgs.get(uuid) );
}
}
@@ -531,4 +540,3 @@ public class ExportAdmins extends ExportingToolBase {
}
}
}
-
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5ebb375/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgInterface.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgInterface.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgInterface.java
new file mode 100644
index 0000000..2debd4d
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgInterface.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import rx.Observable;
+
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+
+/**
+ * Mockable abstraction of user-org management.
+ */
+interface UserOrgInterface {
+
+ Observable<Org> getOrgs() throws Exception;
+
+ Observable<OrgUser> getUsers() throws Exception;
+
+ Set<Org> getUsersOrgs(OrgUser user) throws Exception;
+
+ Set<OrgUser> getOrgUsers(Org org ) throws Exception;
+
+ void removeOrg(Org keeper, Org duplicate) throws Exception;
+
+ void removeUserFromOrg( OrgUser user, Org org ) throws Exception;
+
+ void addUserToOrg( OrgUser user, Org org ) throws Exception;
+
+ Set<UUID> getOrgApps(Org org) throws Exception;
+
+ void removeAppFromOrg( UUID appId, Org org ) throws Exception;
+
+ void addAppToOrg( UUID appId, Org org ) throws Exception;
+
+ void logDuplicates(Map<String, Set<Org>> duplicatesByName);
+
+ Org getOrg(UUID id ) throws Exception;
+
+ OrgUser getOrgUser(UUID id ) throws Exception;
+
+ OrgUser lookupOrgUserByUsername( String username ) throws Exception;
+
+ OrgUser lookupOrgUserByEmail( String email ) throws Exception;
+
+ void removeOrgUser( OrgUser orgUser ) throws Exception;
+
+ void updateOrgUser(OrgUser targetUserEntity) throws Exception;
+
+ void setOrgUserName(OrgUser other, String newUserName) throws Exception;
+
+ Org selectBest( Set<Org> candidates ) throws Exception;
+
+ class Org implements Comparable<Org> {
+ private UUID id;
+ private String name;
+ private long created;
+ public Object sourceValue;
+
+ public Org( UUID id, String name, long created) {
+ this.id = id;
+ this.name = name;
+ this.created = created;
+ this.created = System.currentTimeMillis();
+ }
+
+ public Org( UUID id, String name) {
+ this( id, name, System.currentTimeMillis());
+ }
+
+ @Override
+ public boolean equals( Object o ) {
+ if ( o instanceof Org ) {
+ Org other = (Org)o;
+ return getId().equals( other.getId() );
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(Org o) {
+ return Long.compare( this.created, o.created );
+ }
+
+ public UUID getId() {
+ return id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public long getCreated() {
+ return created;
+ }
+ }
+
+ class OrgUser implements Comparable<OrgUser> {
+ private UUID id;
+ private String username;
+ private String email;
+ private long created;
+ public Object sourceValue;
+
+ public OrgUser( UUID id, String name, String email, long created ) {
+ this.id = id;
+ this.username = name;
+ this.email = email;
+ this.created = created;
+ }
+
+ public OrgUser( UUID id, String name, String email ) {
+ this( id, name, email, System.currentTimeMillis());
+ }
+
+ public UUID getId() {
+ return id;
+ }
+
+ public String getEmail() {
+ return email;
+ }
+
+ public String getUsername() {
+ return username;
+ }
+
+ public void setUsername( String username ) {
+ this.username = username;
+ }
+
+ public long getCreated() {
+ return created;
+ }
+
+ @Override
+ public boolean equals( Object obj ) {
+ if (obj == null) { return false; }
+ if (obj == this) { return true; }
+ if (obj.getClass() != getClass()) {
+ return false;
+ }
+ OrgUser rhs = (OrgUser) obj;
+ return new EqualsBuilder().appendSuper(super.equals(obj))
+ .append(id, rhs.id)
+ .append(username, rhs.username)
+ .append(email, rhs.email)
+ .append(created, rhs.created)
+ .isEquals();
+ }
+
+ @Override
+ public int compareTo(OrgUser o) {
+ return Long.compare( this.created, o.created );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5ebb375/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgManager.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgManager.java
new file mode 100644
index 0000000..7f973ce
--- /dev/null
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgManager.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+import com.google.common.collect.BiMap;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.entities.Group;
+import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import rx.Observable;
+import rx.Subscriber;
+
+import java.util.*;
+
+
+class UserOrgManager implements UserOrgInterface {
+
+ static final Logger logger = LoggerFactory.getLogger( UserOrgManager.class );
+
+ EntityManagerFactory emf;
+ ManagementService managementService;
+
+ public UserOrgManager(EntityManagerFactory emf, ManagementService managementService) {
+ this.emf = emf;
+ this.managementService = managementService;
+ }
+
+ @Override
+ public Observable<Org> getOrgs() throws Exception {
+
+ return Observable.create( new Observable.OnSubscribe<Org>() {
+
+ @Override
+ public void call(Subscriber<? super Org> subscriber) {
+ subscriber.onStart();
+ try {
+ int count = 0;
+
+ Query query = new Query();
+ query.setLimit( ToolBase.MAX_ENTITY_FETCH );
+ query.setResultsLevel( Query.Level.ALL_PROPERTIES );
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ Results results = em.searchCollection( em.getApplicationRef(), "groups", query );
+
+ while (results.size() > 0) {
+ for (Entity orgEntity : results.getList()) {
+
+ Org org = new Org(
+ orgEntity.getUuid(),
+ orgEntity.getProperty( "path" ) + "",
+ orgEntity.getCreated() );
+ org.sourceValue = orgEntity;
+
+ subscriber.onNext( org );
+
+ if (count++ % 1000 == 0) {
+ logger.info( "Emitted {} orgs", count );
+ }
+
+ }
+ if (results.getCursor() == null) {
+ break;
+ }
+ query.setCursor( results.getCursor() );
+ results = em.searchCollection( em.getApplicationRef(), "groups", query );
+ }
+
+ } catch (Exception e) {
+ subscriber.onError( e );
+ }
+ subscriber.onCompleted();
+ }
+ } );
+ }
+
+ @Override
+ public Observable<OrgUser> getUsers() throws Exception {
+
+ return Observable.create( new Observable.OnSubscribe<OrgUser>() {
+
+ @Override
+ public void call(Subscriber<? super OrgUser> subscriber) {
+ subscriber.onStart();
+ try {
+ int count = 0;
+
+ Query query = new Query();
+ query.setLimit( ToolBase.MAX_ENTITY_FETCH );
+ query.setResultsLevel( Query.Level.ALL_PROPERTIES );
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ Results results = em.searchCollection( em.getApplicationRef(), "users", query );
+
+ while (results.size() > 0) {
+ for (Entity entity : results.getList()) {
+
+ OrgUser orgUser = new OrgUser(
+ entity.getUuid(),
+ entity.getProperty( "username" ) + "",
+ entity.getProperty( "email" ) + "",
+ entity.getCreated()
+ );
+ orgUser.sourceValue = entity;
+
+ subscriber.onNext( orgUser );
+
+ if (count++ % 1000 == 0) {
+ logger.info( "Emitted {} users", count );
+ }
+ }
+ if (results.getCursor() == null) {
+ break;
+ }
+ query.setCursor( results.getCursor() );
+ results = em.searchCollection( em.getApplicationRef(), "users", query );
+ }
+
+ } catch (Exception e) {
+ subscriber.onError( e );
+ }
+ subscriber.onCompleted();
+ }
+ } );
+ }
+
+ @Override
+ public Set<Org> getUsersOrgs(OrgUser user) throws Exception {
+
+ Set<Org> ret = new HashSet<Org>();
+
+ Map<String, Object> orgData = managementService.getAdminUserOrganizationData( user.getId() );
+
+ Map<String, Object> orgs = (Map<String, Object>) orgData.get( "organizations" );
+ for (String orgName : orgs.keySet()) {
+
+ Map<String, Object> orgMap = (Map<String, Object>) orgs.get( orgName );
+ Group group = managementService.getOrganizationProps(
+ UUID.fromString( orgMap.get( "uuid" ).toString() ) );
+
+ Org org = new Org(
+ group.getUuid(),
+ group.getPath(),
+ group.getCreated()
+ );
+ ret.add( org );
+ }
+
+ return ret;
+ }
+
+
+ @Override
+ public void removeOrg(Org keeper, Org duplicate) throws Exception {
+
+ // rename org so that it is no longer a duplicate
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ em.delete( new SimpleEntityRef( "group", duplicate.getId() ) );
+ logger.info( "Deleted org {}:{}", new Object[]{duplicate.getName(), duplicate.getId()} );
+
+ // fix the org name index
+ OrganizationInfo orgInfoKeeper = managementService.getOrganizationByUuid( keeper.getId() );
+ try {
+ Entity orgKeeper = em.get( keeper.getId() );
+ em.update( orgKeeper );
+ //managementService.updateOrganizationUniqueIndex( orgInfoKeeper, duplicate.getId() );
+ logger.info( "Updated index for keeper {}:{} not dup {}", new Object[]{
+ orgInfoKeeper.getName(), orgInfoKeeper.getUuid(), duplicate.getId()} );
+
+ } catch (Exception e) {
+ // if there are multiple duplicates this will fail for all but one of them. That's OK
+ logger.warn( "Error repairing unique value keeper {} duplicate {}",
+ keeper.getId(), duplicate.getId() );
+ }
+ }
+
+
+ @Override
+ public Set<OrgUser> getOrgUsers(Org org) throws Exception {
+
+ Set<OrgUser> ret = new HashSet<OrgUser>();
+
+ List<UserInfo> userInfos = managementService.getAdminUsersForOrganization( org.getId() );
+
+ for (UserInfo userInfo : userInfos) {
+ OrgUser orgUser = new OrgUser( userInfo.getUuid(), userInfo.getUsername(), userInfo.getEmail() );
+ ret.add( orgUser );
+ }
+
+ return ret;
+ }
+
+
+ @Override
+ public void removeUserFromOrg(OrgUser user, Org org) throws Exception {
+ // forcefully remove admin user from org
+ managementService.removeAdminUserFromOrganization( user.getId(), org.getId(), true );
+ logger.info( "Removed user {}:{} from org {}:{}", new Object[]{
+ user.getUsername(), user.getId(), org.getName(), org.getId()} );
+ }
+
+
+ @Override
+ public void addUserToOrg(OrgUser user, Org org) throws Exception {
+ UserInfo userInfo = managementService.getAdminUserByUsername( user.getUsername() );
+ OrganizationInfo orgInfo = managementService.getOrganizationByUuid( org.getId() );
+ managementService.addAdminUserToOrganization( userInfo, orgInfo, false );
+ logger.info( "Added user {}:{} to org {}:{}", new Object[]{
+ user.getUsername(), user.getId(), org.getName(), org.getId()} );
+ }
+
+
+ @Override
+ public Set<UUID> getOrgApps(Org org) throws Exception {
+ BiMap<UUID, String> apps = managementService.getApplicationsForOrganization( org.getId() );
+ return apps.keySet();
+ }
+
+
+ @Override
+ public void removeAppFromOrg(UUID appId, Org org) throws Exception {
+ managementService.removeOrganizationApplication( org.getId(), appId );
+ logger.info( "Removed app {} from org {}:{}", new Object[]{
+ appId, org.getName(), org.getId()} );
+ }
+
+
+ @Override
+ public void addAppToOrg(UUID appId, Org org) throws Exception {
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ Entity appEntity = em.getApplication();
+ managementService.addApplicationToOrganization( org.getId(), appEntity );
+ logger.info( "Added app {} to org {}:{}", new Object[]{
+ appId, org.getName(), org.getId()} );
+ }
+
+
+ @Override
+ public void logDuplicates(Map<String, Set<Org>> duplicatesByName) {
+
+ for (String orgName : duplicatesByName.keySet()) {
+ Set<Org> orgs = duplicatesByName.get( orgName );
+ for (Org org : orgs) {
+ Entity orgEntity = (Entity) org.sourceValue;
+
+ StringBuilder sb = new StringBuilder();
+ sb.append( orgEntity.toString() ).append( ", " );
+
+ try {
+ BiMap<UUID, String> apps =
+ managementService.getApplicationsForOrganization( orgEntity.getUuid() );
+ String sep = "";
+ for (UUID uuid : apps.keySet()) {
+ String appName = apps.get( uuid );
+ sb.append( appName ).append( ":" ).append( uuid ).append( sep );
+ sep = ", ";
+ }
+
+ } catch (Exception e) {
+ logger.error( "Error getting applications for org {}:{}", org.getName(), org.getId() );
+ }
+
+ logger.info( sb.toString() );
+ }
+ }
+ }
+
+
+ @Override
+ public Org getOrg(UUID uuid) throws Exception {
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ Group entity = em.get( uuid , Group.class );
+ if ( entity != null ) {
+ Org org = new Org(
+ entity.getUuid(),
+ entity.getPath(),
+ entity.getCreated() );
+ org.sourceValue = entity;
+ return org;
+ }
+ return null;
+ }
+
+
+ @Override
+ public OrgUser getOrgUser(UUID uuid) throws Exception {
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ User entity = em.get( uuid, User.class );
+ if ( entity != null ) {
+ OrgUser user = new OrgUser(
+ entity.getUuid(),
+ entity.getUsername(),
+ entity.getEmail(),
+ entity.getCreated()
+ );
+ return user;
+ }
+ return null;
+ }
+
+
+ @Override
+ public OrgUser lookupOrgUserByUsername(String username) throws Exception {
+ UserInfo info = managementService.getAdminUserByUsername( username );
+ return info == null ? null : getOrgUser( info.getUuid() );
+ }
+
+
+ @Override
+ public OrgUser lookupOrgUserByEmail(String email) throws Exception {
+ UserInfo info = managementService.getAdminUserByEmail( email );
+ return info == null ? null : getOrgUser( info.getUuid() );
+ }
+
+
+ @Override
+ public void removeOrgUser(OrgUser orgUser) throws Exception {
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ em.delete( new SimpleEntityRef( "user", orgUser.getId() ));
+ }
+
+
+ @Override
+ public void updateOrgUser(OrgUser targetUserEntity ) throws Exception {
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ User user = em.get(targetUserEntity.getId(), User.class);
+ user.setUsername( targetUserEntity.getUsername() );
+ user.setEmail( targetUserEntity.getEmail() );
+ em.update( user );
+ }
+
+
+ @Override
+ public void setOrgUserName(OrgUser other, String newUserName ) throws Exception {
+
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+
+ logger.info( "Setting username to {} for user with username {} and id {}", new Object[] {
+ newUserName, other.getUsername(), other.getId()
+ } );
+
+ try {
+ em.setProperty( new SimpleEntityRef( "user", other.getId() ), "username", newUserName, true );
+ }
+ catch ( DuplicateUniquePropertyExistsException e ) {
+ logger.warn( "More than 1 user has the username of {}. Setting the username to their username+UUID as a "
+ + "fallback", newUserName );
+
+ setOrgUserName( other, String.format( "%s-%s", other.getUsername(), other.getId() ) );
+ }
+ }
+
+
+ /**
+ * Select best org from a set of duplicates by picking the one that is indexed, or the oldest.
+ */
+ @Override
+ public Org selectBest(Set<Org> orgs) throws Exception {
+ Org oldest = null;
+ for ( Org org :orgs ) {
+ OrganizationInfo info = managementService.getOrganizationByName( org.getName() );
+ if ( info != null ) {
+ return org;
+ }
+ if ( oldest == null || org.compareTo( oldest ) < 0 ) {
+ oldest = org;
+ }
+ }
+ return oldest;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5ebb375/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
index 2946ddb..6009968 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
@@ -34,10 +34,6 @@ import static org.junit.Assert.assertTrue;
public class ExportAppTest extends AbstractServiceIT {
static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
- int NUM_COLLECTIONS = 10;
- int NUM_ENTITIES = 50;
- int NUM_CONNECTIONS = 3;
-
@org.junit.Test
public void testBasicOperation() throws Exception {
[2/3] usergrid git commit: Revert ""port" tools from master"
Posted by sn...@apache.org.
Revert ""port" tools from master"
This reverts commit e5ebb375386f34b2ad8883c28701500b59208a81.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7689f771
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7689f771
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7689f771
Branch: refs/heads/release
Commit: 7689f7717e6a42b06a247b365891a92931c66c48
Parents: e5ebb37
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jan 22 09:33:05 2016 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jan 22 09:33:05 2016 -0500
----------------------------------------------------------------------
.../usergrid/management/ManagementService.java | 4 +-
.../cassandra/ManagementServiceImpl.java | 10 +-
.../usergrid/tools/DryRunUserOrgManager.java | 62 ---
.../usergrid/tools/DuplicateAdminRepair.java | 293 -------------
.../usergrid/tools/DuplicateOrgRepair.java | 421 -------------------
.../org/apache/usergrid/tools/ExportAdmins.java | 38 +-
.../apache/usergrid/tools/UserOrgInterface.java | 173 --------
.../apache/usergrid/tools/UserOrgManager.java | 389 -----------------
.../apache/usergrid/tools/ExportAppTest.java | 4 +
9 files changed, 22 insertions(+), 1372 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7689f771/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 0851838..cf2924b 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -229,9 +229,7 @@ public interface ManagementService {
String objectType, String objectName, String title, String content )
throws Exception;
- public void removeAdminUserFromOrganization(UUID userId, UUID organizationId ) throws Exception;
-
- public void removeAdminUserFromOrganization(UUID userId, UUID organizationId, boolean force ) throws Exception;
+ public void removeAdminUserFromOrganization( UUID userId, UUID organizationId ) throws Exception;
public void removeOrganizationApplication( UUID organizationId, UUID applicationId ) throws Exception;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7689f771/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index fe8152d..f252705 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -1585,13 +1585,7 @@ public class ManagementServiceImpl implements ManagementService {
@Override
- public void removeAdminUserFromOrganization(UUID userId, UUID organizationId ) throws Exception {
- removeAdminUserFromOrganization( userId, organizationId, false );
- }
-
-
- @Override
- public void removeAdminUserFromOrganization(UUID userId, UUID organizationId, boolean force) throws Exception {
+ public void removeAdminUserFromOrganization( UUID userId, UUID organizationId ) throws Exception {
if ( ( userId == null ) || ( organizationId == null ) ) {
return;
@@ -1600,7 +1594,7 @@ public class ManagementServiceImpl implements ManagementService {
EntityManager em = emf.getEntityManager( smf.getManagementAppId() );
try {
- if ( !force && em.getCollection( new SimpleEntityRef( Group.ENTITY_TYPE, organizationId ), "users", null, 2,
+ if ( em.getCollection( new SimpleEntityRef( Group.ENTITY_TYPE, organizationId ), "users", null, 2,
Level.IDS, false ).size() <= 1 ) {
throw new Exception();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7689f771/stack/tools/src/main/java/org/apache/usergrid/tools/DryRunUserOrgManager.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DryRunUserOrgManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DryRunUserOrgManager.java
deleted file mode 100644
index 966b18c..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/DryRunUserOrgManager.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-import org.apache.usergrid.management.ManagementService;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-
-import java.util.UUID;
-
-class DryRunUserOrgManager extends UserOrgManager {
-
- public DryRunUserOrgManager(EntityManagerFactory emf, ManagementService managementService) {
- super( emf, managementService );
- }
-
- @Override
- public void removeUserFromOrg(OrgUser user, Org org) throws Exception {
- }
-
- @Override
- public void addUserToOrg(OrgUser user, Org org) throws Exception {
- }
-
- @Override
- public void addAppToOrg(UUID appId, Org org) throws Exception {
- }
-
- @Override
- public void removeOrgUser(OrgUser orgUser) {
- }
-
- @Override
- public void updateOrgUser(OrgUser targetUserEntity) {
- }
-
- @Override
- public void setOrgUserName(OrgUser other, String newUserName) {
-
- }
-
- @Override
- public void removeAppFromOrg(UUID appId, Org org) throws Exception {
- }
-
- @Override
- public void removeOrg(Org keeper, Org duplicate) throws Exception {
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7689f771/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateAdminRepair.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateAdminRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateAdminRepair.java
deleted file mode 100644
index 172199c..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateAdminRepair.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.Multimap;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.usergrid.management.ManagementService;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import rx.functions.Action1;
-
-import java.util.*;
-
-import static org.apache.usergrid.tools.UserOrgInterface.Org;
-import static org.apache.usergrid.tools.UserOrgInterface.OrgUser;
-
-
-/**
- * Find duplicate admin users, delete the one that is not indexed.
- */
-public class DuplicateAdminRepair extends ToolBase {
-
- UserOrgInterface manager = null;
-
- static final String THREADS_ARG_NAME = "threads";
-
- int threadCount = 5;
-
- static final String DRYRUN_ARG_NAME = "dryrun";
-
- boolean dryRun = false;
-
- Multimap<String, OrgUser> emails = HashMultimap.create();
-
- Multimap<String, OrgUser> usernames = HashMultimap.create();
-
- boolean testing = false;
-
-
- DuplicateAdminRepair() {
- super();
- }
-
- DuplicateAdminRepair(EntityManagerFactory emf, ManagementService managementService ) {
- this();
- this.emf = emf;
- this.managementService = managementService;
- }
-
-
- @Override
- @SuppressWarnings("static-access")
- public Options createOptions() {
-
- Options options = super.createOptions();
-
- Option dryRunOption = OptionBuilder.hasArg()
- .withType(Boolean.TRUE)
- .withDescription( "-" + DRYRUN_ARG_NAME + " true to print what tool would do and do not alter data.")
- .create( DRYRUN_ARG_NAME );
- options.addOption( dryRunOption );
-
- Option writeThreadsOption = OptionBuilder.hasArg()
- .withType(0)
- .withDescription( "Write Threads -" + THREADS_ARG_NAME )
- .create(THREADS_ARG_NAME);
- options.addOption( writeThreadsOption );
-
- return options;
- }
-
-
- public UserOrgManager createNewRepairManager() {
- return new UserOrgManager( emf, managementService );
- }
-
-
- @Override
- public void runTool(CommandLine line) throws Exception {
-
- startSpring();
- setVerbose( line );
-
- if (StringUtils.isNotEmpty( line.getOptionValue( THREADS_ARG_NAME ) )) {
- try {
- threadCount = Integer.parseInt( line.getOptionValue( THREADS_ARG_NAME ) );
- } catch (NumberFormatException nfe) {
- logger.error( "-" + THREADS_ARG_NAME + " must be specified as an integer. Aborting..." );
- return;
- }
- }
-
- if ( StringUtils.isNotEmpty( line.getOptionValue( DRYRUN_ARG_NAME ) )) {
- dryRun = Boolean.parseBoolean( line.getOptionValue( DRYRUN_ARG_NAME ));
- }
-
- if ( manager == null ) { // we use a special manager when mockTesting
- if (dryRun) {
- manager = new DryRunUserOrgManager( emf, managementService );
- } else {
- manager = new UserOrgManager( emf, managementService );
- }
- }
-
- logger.info( "DuplicateAdminUserRepair tool starting up... manager: " + manager.getClass().getSimpleName() );
-
- // build multi-map of users by email and users by name
-
- manager.getUsers().doOnNext( new Action1<OrgUser>() {
- @Override
- public void call( OrgUser user ) {
-
- if (user.getUsername() == null) {
- logger.warn( "User {} has no username", user.getId() );
- return;
- }
- if (user.getEmail() == null) {
- logger.warn( "User {} has no email", user.getId() );
- return;
- }
- emails.put( user.getEmail().toLowerCase(), user );
- usernames.put( user.getUsername().toLowerCase(), user );
-
- }
- } ).toBlocking().lastOrDefault( null );
-
- for ( String username : usernames.keySet() ) {
- Collection<OrgUser> users = usernames.get( username );
-
- if ( users.size() > 1 ) {
- logger.info( "Found multiple users with the username {}", username );
-
- // force the username to be reset to the user's email
- resolveUsernameConflicts( username, users );
- }
- }
-
- for ( String email : emails.keySet() ) {
-
- // go through a set of users with duplicate emails
- Collection<OrgUser> usersWithDupEmails = emails.get( email );
-
- if ( usersWithDupEmails.size() > 1 ) {
-
- // get the user that is in the users-by-email index, it's the keeper
- OrgUser indexedUser = manager.lookupOrgUserByEmail( email );
-
- if ( indexedUser == null ) {
-
- // no user is indexed with that email, pick oldest as keeper
- List<OrgUser> tempUsers = new ArrayList<OrgUser>( usersWithDupEmails );
- Collections.sort( tempUsers );
- OrgUser oldestOfLot = tempUsers.get( 0 );
-
- logger.warn( "Could not load target user by email {}, loading by UUID {} instead", email, oldestOfLot );
- indexedUser = oldestOfLot;
- }
-
- logger.warn( "Found multiple admins with the email {}. Retaining uuid {}", email, indexedUser.getId() );
-
- for ( OrgUser orgUser : usersWithDupEmails ) {
- if ( !orgUser.getId().equals( indexedUser.getId() )) {
- mergeAdmins( orgUser, indexedUser );
- }
- }
-
- // force the index update after all other admins have been merged
- if ( dryRun ) {
- logger.info("Would force re-index of 'keeper' user {}:{}",
- indexedUser.getUsername(), indexedUser.getId());
- } else {
- logger.info( "Forcing re-index of admin with email {} and id {}", email, indexedUser.getId());
- manager.updateOrgUser( indexedUser );
- }
- }
- }
-
- logger.info( "Repair complete" );
- }
-
-
- /**
- * When our usernames are equal, we need to check if our emails are equal. If they're not, we need to change the one
- * that DOES NOT get returned on a lookup by username
- */
- private void resolveUsernameConflicts( String userName, Collection<OrgUser> users ) throws Exception {
-
- // lookup the admin id
- OrgUser existing = manager.lookupOrgUserByUsername( userName );
-
- if ( existing == null ) {
- logger.warn( "Could not determine an admin for colliding username '{}'. Skipping", userName );
- return;
- }
-
- users.remove( existing );
-
- boolean collision = false;
-
- for ( OrgUser other : users ) {
-
- // same username and email, these will be merged later in the process,
- // skip it
- if ( other != null && other.getEmail() != null && other.getEmail().equals( existing.getEmail() ) ) {
- logger.info(
- "Users with the same username '{}' have the same email '{}'. This will be resolved later in "
- + "the process, skipping", userName, existing.getEmail() );
- continue;
- }
-
- // if we get here, the emails do not match, but the usernames do. Force
- // both usernames to emails
- collision = true;
-
- setUserName(other, other.getEmail() );
- }
-
- if ( collision ) {
- setUserName(existing, existing.getEmail() );
- }
- }
-
-
- /** Set the username to the one provided, if we can't due to duplicate property issues, we fall back to user+uuid */
- private void setUserName( OrgUser other, String newUserName ) throws Exception {
-
- if ( dryRun ) {
- logger.info("Would rename user {}:{} to {}", new Object[] {
- other.getUsername(), other.getId(), newUserName });
- } else {
- logger.info( "Setting username to {} for user {}:{}", new Object[] {
- newUserName, other.getUsername(), other.getId() } );
-
- manager.setOrgUserName( other, newUserName );
- }
- }
-
-
- /** Merge the source admin to the target admin by copying oranizations. Then deletes the source admin */
- private void mergeAdmins( OrgUser sourceUser, OrgUser targetUser ) throws Exception {
-
- logger.info("---> Merging user {}:{} into {}:{}", new Object[] {
- sourceUser.getEmail(), sourceUser.getId(), targetUser.getEmail(), targetUser.getId()
- } );
-
- Set<Org> sourceOrgs = manager.getUsersOrgs( sourceUser );
-
- for ( Org org : sourceOrgs ) {
-
- if ( dryRun ) {
- logger.info( "Would add organization {}:{} to admin with email {} and id {}",
- new Object[] { org.getName(), org.getId(), targetUser.getEmail(), targetUser.getId() } );
-
- } else {
- logger.info( "Adding organization {}:{} to admin with email {} and id {}",
- new Object[] { org.getName(), org.getId(), targetUser.getEmail(), targetUser.getId() } );
-
- // add targetUser to sourceUser's org
- manager.addUserToOrg( targetUser, org );
- }
- }
-
- if ( dryRun ) {
- logger.info( "Would remove admin with email {} and id {} from system",
- sourceUser.getEmail(), sourceUser.getId() );
-
- } else {
- logger.info( "Removing admin with email {} and id {} from system",
- sourceUser.getEmail(), sourceUser.getId() );
-
- manager.removeOrgUser( sourceUser );
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7689f771/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
deleted file mode 100644
index 0f93ed4..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.usergrid.management.ManagementService;
-import org.apache.usergrid.persistence.*;
-import rx.Scheduler;
-import rx.functions.Action1;
-import rx.schedulers.Schedulers;
-
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-import static org.apache.usergrid.tools.UserOrgInterface.Org;
-import static org.apache.usergrid.tools.UserOrgInterface.OrgUser;
-
-/**
- * Find duplicate orgs, delete all but oldest of each and assign users to it.
- */
-public class DuplicateOrgRepair extends ToolBase {
-
- UserOrgInterface manager = null;
-
- Map<String, Set<Org>> orgsByName = new HashMap<String, Set<Org>>();
-
- Map<UUID, Org> orgsById = new HashMap<UUID, Org>();
-
- Map<OrgUser, Set<Org>> orgsByUser = new HashMap<OrgUser, Set<Org>>();
-
- Map<String, Set<Org>> duplicatesByName = new HashMap<String, Set<Org>>();
-
- static final String THREADS_ARG_NAME = "threads";
-
- int threadCount = 5;
-
- static final String DRYRUN_ARG_NAME = "dryrun";
-
- boolean dryRun = false;
-
- static final String NO_AUGMENT_ARG_NAME = "noaugment";
-
- boolean noAugment = false;
-
- static final String ORG1_ID = "org1";
-
- static final String ORG2_ID = "org2";
-
- boolean testing = false;
-
-
- DuplicateOrgRepair() {
- super();
- }
-
- DuplicateOrgRepair( EntityManagerFactory emf, ManagementService managementService ) {
- this();
- this.emf = emf;
- this.managementService = managementService;
- }
-
- @Override
- @SuppressWarnings("static-access")
- public Options createOptions() {
-
- Options options = super.createOptions();
-
- Option dryRunOption = OptionBuilder.hasArg()
- .withType(Boolean.TRUE)
- .withDescription( "-" + DRYRUN_ARG_NAME + " true to print what tool would do and do not alter data.")
- .create( DRYRUN_ARG_NAME );
- options.addOption( dryRunOption );
-
- Option noAugmentRunOption = OptionBuilder.hasArg()
- .withType(Boolean.TRUE)
- .withDescription( "-" + NO_AUGMENT_ARG_NAME + " true to skip the augment step.")
- .create( NO_AUGMENT_ARG_NAME );
- options.addOption( noAugmentRunOption );
-
-
- Option writeThreadsOption = OptionBuilder.hasArg()
- .withType(0)
- .withDescription( "Write Threads -" + THREADS_ARG_NAME )
- .create(THREADS_ARG_NAME);
- options.addOption( writeThreadsOption );
-
- Option org1Option = OptionBuilder.hasArg()
- .withType(0)
- .withDescription( "Duplicate org #1 id -" + ORG1_ID)
- .create(ORG1_ID);
- options.addOption( org1Option );
-
- Option org2Option = OptionBuilder.hasArg()
- .withType(0)
- .withDescription( "Duplicate org #2 id -" + ORG2_ID)
- .create(ORG2_ID);
- options.addOption( org2Option );
-
- return options;
- }
-
- @Override
- public void runTool(CommandLine line) throws Exception {
-
- startSpring();
- setVerbose( line );
-
- UUID org1uuid = null;
- UUID org2uuid = null;
-
- String org1string = line.getOptionValue( ORG1_ID );
- String org2string = line.getOptionValue( ORG2_ID );
-
- if ( org1string != null && org2string == null ) {
- logger.error("- if {} is specified you must also specify {} and vice-versa", ORG1_ID, ORG2_ID);
- return;
-
- } else if ( org2string != null && org1string == null ) {
- logger.error("- if {} is specified you must also specify {} and vice-versa", ORG2_ID, ORG1_ID);
- return;
-
- } else if ( org1string != null && org2string != null ) {
-
- try {
- org1uuid = UUID.fromString( org1string );
- org2uuid = UUID.fromString( org2string );
- } catch (Exception e) {
- logger.error("{} and {} must be specified as UUIDs", ORG1_ID, ORG2_ID);
- return;
- }
- }
-
- if (StringUtils.isNotEmpty( line.getOptionValue( THREADS_ARG_NAME ) )) {
- try {
- threadCount = Integer.parseInt( line.getOptionValue( THREADS_ARG_NAME ) );
- } catch (NumberFormatException nfe) {
- logger.error( "-" + THREADS_ARG_NAME + " must be specified as an integer. Aborting..." );
- return;
- }
- }
-
- if ( StringUtils.isNotEmpty( line.getOptionValue( DRYRUN_ARG_NAME ) )) {
- dryRun = Boolean.parseBoolean( line.getOptionValue( DRYRUN_ARG_NAME ));
- }
-
- if ( StringUtils.isNotEmpty( line.getOptionValue( NO_AUGMENT_ARG_NAME ) )) {
- noAugment = Boolean.parseBoolean( line.getOptionValue( NO_AUGMENT_ARG_NAME ));
- }
-
- if ( manager == null ) { // we use a special manager when mockTesting
- if (dryRun) {
- manager = new DryRunUserOrgManager( emf, managementService );
- } else {
- manager = new UserOrgManager( emf, managementService );
- }
- }
-
- logger.info( "DuplicateOrgRepair tool starting up... manager: " + manager.getClass().getSimpleName() );
-
- if ( org1uuid != null && org2uuid != null ) {
-
- Org org1 = manager.getOrg( org1uuid );
- Org org2 = manager.getOrg( org2uuid );
-
- if ( org1.getName().equalsIgnoreCase( org2.getName() )) {
- buildOrgMaps( org1, org2 );
- } else {
- logger.error("org1 and org2 do not have same duplicate name");
- return;
- }
-
- } else {
- buildOrgMaps();
- }
-
- if ( noAugment == false ) {
- augmentUserOrgsMap();
- }
-
- manager.logDuplicates( duplicatesByName );
-
- mergeDuplicateOrgs();
-
- removeDuplicateOrgs();
-
- logger.info( "DuplicateOrgRepair work is done!");
- }
-
-
- public UserOrgManager createNewRepairManager() {
- return new UserOrgManager( emf, managementService );
- }
-
-
- private void buildOrgMaps(Org org1, Org org2) {
-
- Set<Org> orgs = new HashSet<Org>();
- orgs.add( org1 );
- orgs.add( org2 );
- orgsByName.put( org1.getName().toLowerCase(), orgs );
- duplicatesByName.put( org1.getName().toLowerCase(), orgs );
-
- orgsById.put( org1.getId(), org1 );
- orgsById.put( org2.getId(), org2 );
-
- for ( Org org : orgs ) {
- try {
- Set<OrgUser> orgUsers = manager.getOrgUsers( org );
- for (OrgUser user : orgUsers) {
- Set<Org> usersOrgs = orgsByUser.get( user );
- if (usersOrgs == null) {
- usersOrgs = new HashSet<Org>();
- orgsByUser.put( user, usersOrgs );
- }
- usersOrgs.add( org );
- }
- } catch (Exception e) {
- logger.error( "Error getting users for org {}:{}", org.getName(), org.getId() );
- logger.error( "Stack trace is: ", e );
- }
- }
-
- }
-
-
- /**
- * build map of orgs by name, orgs by id, orgs by user and duplicate orgs by name
- */
- private void buildOrgMaps() throws Exception {
-
- manager.getOrgs().doOnNext( new Action1<Org>() {
- @Override
- public void call(Org org) {
-
- // orgs by name and duplicate orgs by name maps
-
- Set<Org> orgs = orgsByName.get( org.getName().toLowerCase() );
- if (orgs == null) {
- orgs = new HashSet<Org>();
- orgsByName.put( org.getName().toLowerCase(), orgs );
- } else {
- duplicatesByName.put( org.getName().toLowerCase(), orgs );
- }
- orgs.add( org );
-
- orgsById.put( org.getId(), org );
-
- // orgs by user map, created via org -> user connections
-
- try {
- Set<OrgUser> orgUsers = manager.getOrgUsers( org );
- for ( OrgUser user : orgUsers ) {
- Set<Org> usersOrgs = orgsByUser.get( user );
- if (usersOrgs == null) {
- usersOrgs = new HashSet<Org>();
- orgsByUser.put( user, usersOrgs );
- }
- usersOrgs.add( org );
- }
- } catch (Exception e) {
- logger.error("Error getting users for org {}:{}", org.getName(), org.getId());
- logger.error("Stack trace is: ", e);
- }
-
- }
-
- } ).toBlocking().lastOrDefault( null );
-
- logger.info( "DuplicateOrgRepair tool built org maps");
- }
-
-
- /**
- * augment user orgs map via user -> org connections
- */
- private void augmentUserOrgsMap() throws Exception {
-
- ExecutorService writeThreadPoolExecutor = Executors.newFixedThreadPool( threadCount );
- Scheduler scheduler = Schedulers.from( writeThreadPoolExecutor );
-
- manager.getUsers().doOnNext( new Action1<OrgUser>() {
- @Override
- public void call(OrgUser user) {
- try {
- Set<Org> connectedToOrgs = manager.getUsersOrgs(user);
- Set<Org> usersOrgs = orgsByUser.get(user);
- if ( usersOrgs == null ) {
- usersOrgs = new HashSet<Org>();
- }
- for ( Org org : connectedToOrgs ) {
- if (!usersOrgs.contains(org)) {
- usersOrgs.add(org);
- }
- }
-
- } catch (Exception e) {
- logger.error("Error getting orgs for user {}:{}", user.getUsername(), user.getId());
- logger.error("Stack trace is: ", e);
- }
- }
- } ).subscribeOn( scheduler ).toBlocking().lastOrDefault( null );
-
- logger.info( "DuplicateOrgRepair augmented user orgs map");
- }
-
-
- /**
- * For each duplicate name, pick best org and merge apps and users into it
- */
- private void mergeDuplicateOrgs() throws Exception {
-
- for ( String dupName : duplicatesByName.keySet() ) {
- Set<Org> duplicateOrgs = duplicatesByName.get(dupName);
- Org bestOrg = manager.selectBest( duplicateOrgs );
-
- for ( Org org : duplicateOrgs ) {
-
- if ( !org.equals( bestOrg )) {
-
- Set<OrgUser> orgUsers = new HashSet<OrgUser>( manager.getOrgUsers( org ) );
-
- for (OrgUser user : orgUsers) {
- if (dryRun) {
- Object[] args = new Object[]{
- user.getUsername(), user.getId(), bestOrg.getName(), bestOrg.getId()};
- logger.info( "Would add user {}:{} to org {}:{}", args );
- args = new Object[]{
- user.getUsername(), user.getId(), org.getName(), org.getId()};
- logger.info( "Would remove user {}:{} org {}:{}", args );
- } else {
- try {
- manager.addUserToOrg( user, bestOrg );
- } catch ( Exception e ) {
- Object[] args = new Object[]{
- user.getUsername(), user.getId(), bestOrg.getName(), bestOrg.getId()};
- logger.error( "Error adding user {}:{} to org {}:{}", args );
- }
- try {
- manager.removeUserFromOrg( user, org );
- } catch ( Exception e ) {
- Object[] args = new Object[]{
- user.getUsername(), user.getId(), org.getName(), org.getId()};
- logger.info( "Error removing user {}:{} org {}:{}", args );
- }
- }
- }
-
- Set<UUID> orgApps = new HashSet<UUID>( manager.getOrgApps( org ) );
-
- for (UUID appId : orgApps) {
- if (dryRun) {
- Object[] args = new Object[]{ appId, bestOrg.getName(), bestOrg.getId()};
- logger.info( "Would add app {} to org {}:{}", args );
- args = new Object[]{ appId, org.getName(), org.getId()};
- logger.info( "Would remove app {} org {}:{}", args );
- } else {
- try {
- manager.addAppToOrg( appId, bestOrg );
- } catch ( Exception e ) {
- Object[] args = new Object[]{ appId, bestOrg.getName(), bestOrg.getId()};
- logger.error( "Error adding app {} to org {}:{}", args );
- }
- try {
- manager.removeAppFromOrg( appId, org );
- } catch (Exception e ) {
- Object[] args = new Object[]{ appId, org.getName(), org.getId()};
- logger.info( "Error removing app {} org {}:{}", args );
- }
- }
- }
-
- }
- }
- }
-
- logger.info( "DuplicateOrgRepair merged duplicate orgs");
- }
-
-
- /**
- * remove/rename duplicate orgs so they no longer impact operation of system
- */
- private void removeDuplicateOrgs() throws Exception {
- for ( String dupName : duplicatesByName.keySet() ) {
- Set<Org> orgs = duplicatesByName.get( dupName );
- Org best = manager.selectBest( orgs );
- for ( Org candidate : orgs ) {
- if ( !candidate.equals(best) ) {
- if ( dryRun ) {
- logger.info("Would rename/remove org {}:{}",
- new Object[] { candidate.getName(), candidate.getId() });
- } else {
- manager.removeOrg( best, candidate );
- }
- }
- }
- }
-
- logger.info( "DuplicateOrgRepair renamed/removed duplicate orgs");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7689f771/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index 070a499..479d740 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@ -16,7 +16,6 @@
*/
package org.apache.usergrid.tools;
-
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import org.apache.commons.cli.CommandLine;
@@ -29,7 +28,6 @@ import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.utils.StringUtils;
import org.codehaus.jackson.JsonGenerator;
import org.slf4j.Logger;
@@ -40,7 +38,6 @@ import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
-
/**
* Export Admin Users and metadata including organizations and passwords.
*
@@ -67,12 +64,11 @@ public class ExportAdmins extends ExportingToolBase {
public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";
// map admin user UUID to list of organizations to which user belongs
- private Map<UUID, List<Org>> userToOrgsMap = new HashMap<UUID, List<Org>>(100000);
+ private Map<UUID, List<Org>> userToOrgsMap = new HashMap<UUID, List<Org>>(50000);
- private Map<String, UUID> orgNameToUUID = new HashMap<String, UUID>(100000);
+ private Map<String, UUID> orgNameToUUID = new HashMap<String, UUID>(50000);
- private Set<UUID> orgsWritten = new HashSet<UUID>(100000);
- private Set<String> orgsNamesWritten = new HashSet<String>(100000);
+ private Set<UUID> orgsWritten = new HashSet<UUID>(50000);
private Set<UUID> duplicateOrgs = new HashSet<UUID>();
@@ -150,12 +146,10 @@ public class ExportAdmins extends ExportingToolBase {
// start read queue workers
BlockingQueue<UUID> readQueue = new LinkedBlockingQueue<UUID>();
- List<AdminUserReader> readers = new ArrayList<AdminUserReader>();
for (int i = 0; i < readThreadCount; i++) {
AdminUserReader worker = new AdminUserReader( readQueue, writeQueue );
Thread readerThread = new Thread( worker, "AdminUserReader-" + i );
readerThread.start();
- readers.add( worker );
}
logger.debug( readThreadCount + " read worker threads started" );
@@ -197,7 +191,7 @@ public class ExportAdmins extends ExportingToolBase {
Options options = super.createOptions();
Option readThreads = OptionBuilder
- .hasArg().withType(0).withDescription("Read Threads -" + READ_THREAD_COUNT).create(READ_THREAD_COUNT);
+ .hasArg().withType(0).withDescription("Read Threads -" + READ_THREAD_COUNT).create(READ_THREAD_COUNT);
options.addOption( readThreads );
return options;
@@ -211,7 +205,7 @@ public class ExportAdmins extends ExportingToolBase {
logger.info( "Building org map" );
- ExecutorService execService = Executors.newFixedThreadPool( this.readThreadCount );
+ ExecutorService execService = Executors.newFixedThreadPool( readThreadCount );
EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
String queryString = "select *";
@@ -338,8 +332,8 @@ public class ExportAdmins extends ExportingToolBase {
String actionTaken = "Processed";
if (ignoreInvalidUsers && (task.orgNamesByUuid.isEmpty()
- || task.dictionariesByName.isEmpty()
- || task.dictionariesByName.get( "credentials" ).isEmpty())) {
+ || task.dictionariesByName.isEmpty()
+ || task.dictionariesByName.get( "credentials" ).isEmpty())) {
actionTaken = "Ignored";
@@ -348,9 +342,9 @@ public class ExportAdmins extends ExportingToolBase {
}
Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty() ?
- 0 : task.dictionariesByName.get( "credentials" ));
+ 0 : task.dictionariesByName.get( "credentials" ));
- logger.warn( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}",
+ logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}",
new Object[]{
actionTaken,
task.adminUser.getProperty( "username" ),
@@ -376,7 +370,7 @@ public class ExportAdmins extends ExportingToolBase {
Set<String> dictionaries = em.getDictionaries( entity );
if ( dictionaries.isEmpty() ) {
- logger.warn("User {}:{} has no dictionaries", task.adminUser.getName(), task.adminUser.getUuid() );
+ logger.error("User {}:{} has no dictionaries", task.adminUser.getName(), task.adminUser.getUuid() );
return;
}
@@ -391,7 +385,7 @@ public class ExportAdmins extends ExportingToolBase {
task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() );
- List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
+ List<Org> orgs = userToOrgsMap.get( task.adminUser.getUuid() );
if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {
@@ -429,12 +423,12 @@ public class ExportAdmins extends ExportingToolBase {
// write one JSON file for management application users
JsonGenerator usersFile =
- getJsonGenerator( createOutputFile( ADMIN_USERS_PREFIX, em.getApplication().getName() ) );
+ getJsonGenerator( createOutputFile( ADMIN_USERS_PREFIX, em.getApplication().getName() ) );
usersFile.writeStartArray();
// write one JSON file for metadata: collections, connections and dictionaries of those users
JsonGenerator metadataFile =
- getJsonGenerator( createOutputFile( ADMIN_USER_METADATA_PREFIX, em.getApplication().getName() ) );
+ getJsonGenerator( createOutputFile( ADMIN_USER_METADATA_PREFIX, em.getApplication().getName() ) );
metadataFile.writeStartObject();
while ( true ) {
@@ -476,8 +470,7 @@ public class ExportAdmins extends ExportingToolBase {
usersFile.writeEndArray();
usersFile.close();
- logger.info( "Exported TOTAL {} admin users and {} organizations, org names = {}",
- new Object[] { userCount.get(), orgsWritten.size(), orgsNamesWritten.size() } );
+ logger.info( "Exported TOTAL {} admin users and {} organizations", userCount.get(), orgsWritten.size() );
}
@@ -530,9 +523,7 @@ public class ExportAdmins extends ExportingToolBase {
jg.writeEndObject();
synchronized (orgsWritten) {
- logger.info("Exported org {}:{}", uuid, orgs.get(uuid));
orgsWritten.add( uuid );
- orgsNamesWritten.add( orgs.get(uuid) );
}
}
@@ -540,3 +531,4 @@ public class ExportAdmins extends ExportingToolBase {
}
}
}
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7689f771/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgInterface.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgInterface.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgInterface.java
deleted file mode 100644
index 2debd4d..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgInterface.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import rx.Observable;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-
-/**
- * Mockable abstraction of user-org management.
- */
-interface UserOrgInterface {
-
- Observable<Org> getOrgs() throws Exception;
-
- Observable<OrgUser> getUsers() throws Exception;
-
- Set<Org> getUsersOrgs(OrgUser user) throws Exception;
-
- Set<OrgUser> getOrgUsers(Org org ) throws Exception;
-
- void removeOrg(Org keeper, Org duplicate) throws Exception;
-
- void removeUserFromOrg( OrgUser user, Org org ) throws Exception;
-
- void addUserToOrg( OrgUser user, Org org ) throws Exception;
-
- Set<UUID> getOrgApps(Org org) throws Exception;
-
- void removeAppFromOrg( UUID appId, Org org ) throws Exception;
-
- void addAppToOrg( UUID appId, Org org ) throws Exception;
-
- void logDuplicates(Map<String, Set<Org>> duplicatesByName);
-
- Org getOrg(UUID id ) throws Exception;
-
- OrgUser getOrgUser(UUID id ) throws Exception;
-
- OrgUser lookupOrgUserByUsername( String username ) throws Exception;
-
- OrgUser lookupOrgUserByEmail( String email ) throws Exception;
-
- void removeOrgUser( OrgUser orgUser ) throws Exception;
-
- void updateOrgUser(OrgUser targetUserEntity) throws Exception;
-
- void setOrgUserName(OrgUser other, String newUserName) throws Exception;
-
- Org selectBest( Set<Org> candidates ) throws Exception;
-
- class Org implements Comparable<Org> {
- private UUID id;
- private String name;
- private long created;
- public Object sourceValue;
-
- public Org( UUID id, String name, long created) {
- this.id = id;
- this.name = name;
- this.created = created;
- this.created = System.currentTimeMillis();
- }
-
- public Org( UUID id, String name) {
- this( id, name, System.currentTimeMillis());
- }
-
- @Override
- public boolean equals( Object o ) {
- if ( o instanceof Org ) {
- Org other = (Org)o;
- return getId().equals( other.getId() );
- }
- return false;
- }
-
- @Override
- public int compareTo(Org o) {
- return Long.compare( this.created, o.created );
- }
-
- public UUID getId() {
- return id;
- }
-
- public String getName() {
- return name;
- }
-
- public long getCreated() {
- return created;
- }
- }
-
- class OrgUser implements Comparable<OrgUser> {
- private UUID id;
- private String username;
- private String email;
- private long created;
- public Object sourceValue;
-
- public OrgUser( UUID id, String name, String email, long created ) {
- this.id = id;
- this.username = name;
- this.email = email;
- this.created = created;
- }
-
- public OrgUser( UUID id, String name, String email ) {
- this( id, name, email, System.currentTimeMillis());
- }
-
- public UUID getId() {
- return id;
- }
-
- public String getEmail() {
- return email;
- }
-
- public String getUsername() {
- return username;
- }
-
- public void setUsername( String username ) {
- this.username = username;
- }
-
- public long getCreated() {
- return created;
- }
-
- @Override
- public boolean equals( Object obj ) {
- if (obj == null) { return false; }
- if (obj == this) { return true; }
- if (obj.getClass() != getClass()) {
- return false;
- }
- OrgUser rhs = (OrgUser) obj;
- return new EqualsBuilder().appendSuper(super.equals(obj))
- .append(id, rhs.id)
- .append(username, rhs.username)
- .append(email, rhs.email)
- .append(created, rhs.created)
- .isEquals();
- }
-
- @Override
- public int compareTo(OrgUser o) {
- return Long.compare( this.created, o.created );
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7689f771/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgManager.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgManager.java
deleted file mode 100644
index 7f973ce..0000000
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UserOrgManager.java
+++ /dev/null
@@ -1,389 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.tools;
-
-import com.google.common.collect.BiMap;
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.management.ManagementService;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.management.UserInfo;
-import org.apache.usergrid.persistence.*;
-import org.apache.usergrid.persistence.entities.Group;
-import org.apache.usergrid.persistence.entities.User;
-import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import rx.Observable;
-import rx.Subscriber;
-
-import java.util.*;
-
-
-class UserOrgManager implements UserOrgInterface {
-
- static final Logger logger = LoggerFactory.getLogger( UserOrgManager.class );
-
- EntityManagerFactory emf;
- ManagementService managementService;
-
- public UserOrgManager(EntityManagerFactory emf, ManagementService managementService) {
- this.emf = emf;
- this.managementService = managementService;
- }
-
- @Override
- public Observable<Org> getOrgs() throws Exception {
-
- return Observable.create( new Observable.OnSubscribe<Org>() {
-
- @Override
- public void call(Subscriber<? super Org> subscriber) {
- subscriber.onStart();
- try {
- int count = 0;
-
- Query query = new Query();
- query.setLimit( ToolBase.MAX_ENTITY_FETCH );
- query.setResultsLevel( Query.Level.ALL_PROPERTIES );
- EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- Results results = em.searchCollection( em.getApplicationRef(), "groups", query );
-
- while (results.size() > 0) {
- for (Entity orgEntity : results.getList()) {
-
- Org org = new Org(
- orgEntity.getUuid(),
- orgEntity.getProperty( "path" ) + "",
- orgEntity.getCreated() );
- org.sourceValue = orgEntity;
-
- subscriber.onNext( org );
-
- if (count++ % 1000 == 0) {
- logger.info( "Emitted {} orgs", count );
- }
-
- }
- if (results.getCursor() == null) {
- break;
- }
- query.setCursor( results.getCursor() );
- results = em.searchCollection( em.getApplicationRef(), "groups", query );
- }
-
- } catch (Exception e) {
- subscriber.onError( e );
- }
- subscriber.onCompleted();
- }
- } );
- }
-
- @Override
- public Observable<OrgUser> getUsers() throws Exception {
-
- return Observable.create( new Observable.OnSubscribe<OrgUser>() {
-
- @Override
- public void call(Subscriber<? super OrgUser> subscriber) {
- subscriber.onStart();
- try {
- int count = 0;
-
- Query query = new Query();
- query.setLimit( ToolBase.MAX_ENTITY_FETCH );
- query.setResultsLevel( Query.Level.ALL_PROPERTIES );
- EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- Results results = em.searchCollection( em.getApplicationRef(), "users", query );
-
- while (results.size() > 0) {
- for (Entity entity : results.getList()) {
-
- OrgUser orgUser = new OrgUser(
- entity.getUuid(),
- entity.getProperty( "username" ) + "",
- entity.getProperty( "email" ) + "",
- entity.getCreated()
- );
- orgUser.sourceValue = entity;
-
- subscriber.onNext( orgUser );
-
- if (count++ % 1000 == 0) {
- logger.info( "Emitted {} users", count );
- }
- }
- if (results.getCursor() == null) {
- break;
- }
- query.setCursor( results.getCursor() );
- results = em.searchCollection( em.getApplicationRef(), "users", query );
- }
-
- } catch (Exception e) {
- subscriber.onError( e );
- }
- subscriber.onCompleted();
- }
- } );
- }
-
- @Override
- public Set<Org> getUsersOrgs(OrgUser user) throws Exception {
-
- Set<Org> ret = new HashSet<Org>();
-
- Map<String, Object> orgData = managementService.getAdminUserOrganizationData( user.getId() );
-
- Map<String, Object> orgs = (Map<String, Object>) orgData.get( "organizations" );
- for (String orgName : orgs.keySet()) {
-
- Map<String, Object> orgMap = (Map<String, Object>) orgs.get( orgName );
- Group group = managementService.getOrganizationProps(
- UUID.fromString( orgMap.get( "uuid" ).toString() ) );
-
- Org org = new Org(
- group.getUuid(),
- group.getPath(),
- group.getCreated()
- );
- ret.add( org );
- }
-
- return ret;
- }
-
-
- @Override
- public void removeOrg(Org keeper, Org duplicate) throws Exception {
-
- // rename org so that it is no longer a duplicate
- EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- em.delete( new SimpleEntityRef( "group", duplicate.getId() ) );
- logger.info( "Deleted org {}:{}", new Object[]{duplicate.getName(), duplicate.getId()} );
-
- // fix the org name index
- OrganizationInfo orgInfoKeeper = managementService.getOrganizationByUuid( keeper.getId() );
- try {
- Entity orgKeeper = em.get( keeper.getId() );
- em.update( orgKeeper );
- //managementService.updateOrganizationUniqueIndex( orgInfoKeeper, duplicate.getId() );
- logger.info( "Updated index for keeper {}:{} not dup {}", new Object[]{
- orgInfoKeeper.getName(), orgInfoKeeper.getUuid(), duplicate.getId()} );
-
- } catch (Exception e) {
- // if there are multiple duplicates this will fail for all but one of them. That's OK
- logger.warn( "Error repairing unique value keeper {} duplicate {}",
- keeper.getId(), duplicate.getId() );
- }
- }
-
-
- @Override
- public Set<OrgUser> getOrgUsers(Org org) throws Exception {
-
- Set<OrgUser> ret = new HashSet<OrgUser>();
-
- List<UserInfo> userInfos = managementService.getAdminUsersForOrganization( org.getId() );
-
- for (UserInfo userInfo : userInfos) {
- OrgUser orgUser = new OrgUser( userInfo.getUuid(), userInfo.getUsername(), userInfo.getEmail() );
- ret.add( orgUser );
- }
-
- return ret;
- }
-
-
- @Override
- public void removeUserFromOrg(OrgUser user, Org org) throws Exception {
- // forcefully remove admin user from org
- managementService.removeAdminUserFromOrganization( user.getId(), org.getId(), true );
- logger.info( "Removed user {}:{} from org {}:{}", new Object[]{
- user.getUsername(), user.getId(), org.getName(), org.getId()} );
- }
-
-
- @Override
- public void addUserToOrg(OrgUser user, Org org) throws Exception {
- UserInfo userInfo = managementService.getAdminUserByUsername( user.getUsername() );
- OrganizationInfo orgInfo = managementService.getOrganizationByUuid( org.getId() );
- managementService.addAdminUserToOrganization( userInfo, orgInfo, false );
- logger.info( "Added user {}:{} to org {}:{}", new Object[]{
- user.getUsername(), user.getId(), org.getName(), org.getId()} );
- }
-
-
- @Override
- public Set<UUID> getOrgApps(Org org) throws Exception {
- BiMap<UUID, String> apps = managementService.getApplicationsForOrganization( org.getId() );
- return apps.keySet();
- }
-
-
- @Override
- public void removeAppFromOrg(UUID appId, Org org) throws Exception {
- managementService.removeOrganizationApplication( org.getId(), appId );
- logger.info( "Removed app {} from org {}:{}", new Object[]{
- appId, org.getName(), org.getId()} );
- }
-
-
- @Override
- public void addAppToOrg(UUID appId, Org org) throws Exception {
- EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- Entity appEntity = em.getApplication();
- managementService.addApplicationToOrganization( org.getId(), appEntity );
- logger.info( "Added app {} to org {}:{}", new Object[]{
- appId, org.getName(), org.getId()} );
- }
-
-
- @Override
- public void logDuplicates(Map<String, Set<Org>> duplicatesByName) {
-
- for (String orgName : duplicatesByName.keySet()) {
- Set<Org> orgs = duplicatesByName.get( orgName );
- for (Org org : orgs) {
- Entity orgEntity = (Entity) org.sourceValue;
-
- StringBuilder sb = new StringBuilder();
- sb.append( orgEntity.toString() ).append( ", " );
-
- try {
- BiMap<UUID, String> apps =
- managementService.getApplicationsForOrganization( orgEntity.getUuid() );
- String sep = "";
- for (UUID uuid : apps.keySet()) {
- String appName = apps.get( uuid );
- sb.append( appName ).append( ":" ).append( uuid ).append( sep );
- sep = ", ";
- }
-
- } catch (Exception e) {
- logger.error( "Error getting applications for org {}:{}", org.getName(), org.getId() );
- }
-
- logger.info( sb.toString() );
- }
- }
- }
-
-
- @Override
- public Org getOrg(UUID uuid) throws Exception {
- EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- Group entity = em.get( uuid , Group.class );
- if ( entity != null ) {
- Org org = new Org(
- entity.getUuid(),
- entity.getPath(),
- entity.getCreated() );
- org.sourceValue = entity;
- return org;
- }
- return null;
- }
-
-
- @Override
- public OrgUser getOrgUser(UUID uuid) throws Exception {
- EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- User entity = em.get( uuid, User.class );
- if ( entity != null ) {
- OrgUser user = new OrgUser(
- entity.getUuid(),
- entity.getUsername(),
- entity.getEmail(),
- entity.getCreated()
- );
- return user;
- }
- return null;
- }
-
-
- @Override
- public OrgUser lookupOrgUserByUsername(String username) throws Exception {
- UserInfo info = managementService.getAdminUserByUsername( username );
- return info == null ? null : getOrgUser( info.getUuid() );
- }
-
-
- @Override
- public OrgUser lookupOrgUserByEmail(String email) throws Exception {
- UserInfo info = managementService.getAdminUserByEmail( email );
- return info == null ? null : getOrgUser( info.getUuid() );
- }
-
-
- @Override
- public void removeOrgUser(OrgUser orgUser) throws Exception {
- EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- em.delete( new SimpleEntityRef( "user", orgUser.getId() ));
- }
-
-
- @Override
- public void updateOrgUser(OrgUser targetUserEntity ) throws Exception {
- EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- User user = em.get(targetUserEntity.getId(), User.class);
- user.setUsername( targetUserEntity.getUsername() );
- user.setEmail( targetUserEntity.getEmail() );
- em.update( user );
- }
-
-
- @Override
- public void setOrgUserName(OrgUser other, String newUserName ) throws Exception {
-
- EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
-
- logger.info( "Setting username to {} for user with username {} and id {}", new Object[] {
- newUserName, other.getUsername(), other.getId()
- } );
-
- try {
- em.setProperty( new SimpleEntityRef( "user", other.getId() ), "username", newUserName, true );
- }
- catch ( DuplicateUniquePropertyExistsException e ) {
- logger.warn( "More than 1 user has the username of {}. Setting the username to their username+UUID as a "
- + "fallback", newUserName );
-
- setOrgUserName( other, String.format( "%s-%s", other.getUsername(), other.getId() ) );
- }
- }
-
-
- /**
- * Select best org from a set of duplicates by picking the one that is indexed, or the oldest.
- */
- @Override
- public Org selectBest(Set<Org> orgs) throws Exception {
- Org oldest = null;
- for ( Org org :orgs ) {
- OrganizationInfo info = managementService.getOrganizationByName( org.getName() );
- if ( info != null ) {
- return org;
- }
- if ( oldest == null || org.compareTo( oldest ) < 0 ) {
- oldest = org;
- }
- }
- return oldest;
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7689f771/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
index 6009968..2946ddb 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
@@ -34,6 +34,10 @@ import static org.junit.Assert.assertTrue;
public class ExportAppTest extends AbstractServiceIT {
static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
+ int NUM_COLLECTIONS = 10;
+ int NUM_ENTITIES = 50;
+ int NUM_CONNECTIONS = 3;
+
@org.junit.Test
public void testBasicOperation() throws Exception {
[3/3] usergrid git commit: Merge branch 'release' of
https://git-wip-us.apache.org/repos/asf/usergrid into release
Posted by sn...@apache.org.
Merge branch 'release' of https://git-wip-us.apache.org/repos/asf/usergrid into release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/46fb9265
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/46fb9265
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/46fb9265
Branch: refs/heads/release
Commit: 46fb92652b60b7e58f7cc2145042485cbb411189
Parents: 7689f77 62ad5a8
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jan 22 09:36:34 2016 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jan 22 09:36:34 2016 -0500
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 10 +-
.../corepersistence/CpEntityManagerFactory.java | 18 ++-
.../corepersistence/CpRelationManager.java | 53 ++++---
.../usergrid/corepersistence/CpSetup.java | 14 +-
.../read/traverse/AbstractReadGraphFilter.java | 37 ++++-
.../persistence/EntityManagerFactory.java | 8 +-
.../usergrid/persistence/EntityManagerIT.java | 33 ++++
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 18 +--
.../model/entity/MapToEntityConverter.java | 18 ++-
.../persistence/model/field/FieldTypeName.java | 3 +-
.../persistence/model/field/NullField.java | 44 ++++++
.../model/entity/MapToEntityConverterTest.java | 142 ++++++++++++++++++
.../persistence/model/field/EntityTest.java | 9 +-
.../persistence/index/impl/EntityField.java | 11 ++
.../index/impl/EntityMappingParser.java | 16 +-
.../index/impl/EntityToMapConverter.java | 3 +-
.../persistence/index/impl/FieldParser.java | 5 +-
.../persistence/index/impl/IndexingUtils.java | 1 +
.../persistence/index/impl/EntityIndexTest.java | 79 +++++++++-
.../index/impl/EntityToMapConverterTest.java | 149 ++++++++++++++-----
.../index/impl/IndexLoadTestsIT.java | 6 +-
.../management/AppInfoMigrationPlugin.java | 3 +-
.../usergrid/management/ManagementService.java | 4 +-
.../cassandra/ManagementServiceImpl.java | 12 +-
.../migration/AppInfoMigrationPluginTest.java | 2 +-
25 files changed, 564 insertions(+), 134 deletions(-)
----------------------------------------------------------------------