You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2015/12/17 23:14:21 UTC
[2/7] usergrid git commit: Add double protection for dryrun mode,
and some more logging.
Add double protection for dryrun mode, and some more logging.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/de254f0a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/de254f0a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/de254f0a
Branch: refs/heads/1.x
Commit: de254f0acce9f69842f04be3fc0904dae37f54a4
Parents: 674532f
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Dec 11 14:54:26 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Dec 11 14:54:26 2015 -0500
----------------------------------------------------------------------
.../usergrid/tools/DuplicateOrgRepair.java | 102 +++++++++++++++----
.../usergrid/tools/DuplicateOrgRepairTest.java | 6 ++
2 files changed, 87 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de254f0a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
index bc4acb5..4b5e391 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/DuplicateOrgRepair.java
@@ -18,7 +18,6 @@
package org.apache.usergrid.tools;
import com.google.common.collect.BiMap;
-import com.google.common.collect.Sets;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
@@ -51,7 +50,7 @@ import static org.apache.usergrid.tools.DuplicateOrgInterface.OrgUser;
*/
public class DuplicateOrgRepair extends ToolBase {
- DuplicateOrgInterface manager = new Manager();
+ DuplicateOrgInterface manager;
Map<String, Set<Org>> orgsByName = new HashMap<String, Set<Org>>();
@@ -68,6 +67,8 @@ public class DuplicateOrgRepair extends ToolBase {
static final String DRYRUN_ARG_NAME = "dryrun";
static boolean dryRun = false;
+
+ static boolean testing = false;
@Override
@@ -94,13 +95,10 @@ public class DuplicateOrgRepair extends ToolBase {
@Override
public void runTool(CommandLine line) throws Exception {
- logger.info( "DuplicateOrgRepair tool starting up..." );
startSpring();
setVerbose( line );
- dryRun = Boolean.parseBoolean( line.getOptionValue( DRYRUN_ARG_NAME ));
-
if (StringUtils.isNotEmpty( line.getOptionValue( THREADS_ARG_NAME ) )) {
try {
threadCount = Integer.parseInt( line.getOptionValue( THREADS_ARG_NAME ) );
@@ -110,15 +108,27 @@ public class DuplicateOrgRepair extends ToolBase {
}
}
- buildOrgMaps();
+ dryRun = Boolean.parseBoolean( line.getOptionValue( DRYRUN_ARG_NAME ));
+
+ if ( dryRun && !testing ) {
+ manager = new DryRunManager();
+ } else {
+ manager = new RepairManager();
+ }
+ logger.info( "DuplicateOrgRepair tool starting up... manager: " + manager.getClass().getSimpleName() );
+
+ buildOrgMaps();
+
augmentUserOrgsMap();
+ manager.logDuplicates( duplicatesByName );
+
mergeDuplicateOrgs();
-
+
removeDuplicateOrgs();
- logger.info( "DuplicateOrgRepair work is done." );
+ logger.info( "DuplicateOrgRepair work is done!");
}
@@ -158,10 +168,14 @@ public class DuplicateOrgRepair extends ToolBase {
}
} catch (Exception e) {
logger.error("Error getting users for org {}:{}", org.getName(), org.getId());
+ logger.error("Stack trace is: ", e);
}
}
+
} ).toBlocking().lastOrDefault( null );
+
+ logger.info( "DuplicateOrgRepair tool built org maps");
}
@@ -179,6 +193,9 @@ public class DuplicateOrgRepair extends ToolBase {
try {
Set<Org> connectedToOrgs = manager.getUsersOrgs(user);
Set<Org> usersOrgs = orgsByUser.get(user);
+ if ( usersOrgs == null ) {
+ usersOrgs = new HashSet<Org>();
+ }
for ( Org org : connectedToOrgs ) {
if (!usersOrgs.contains(org)) {
usersOrgs.add(org);
@@ -187,9 +204,12 @@ public class DuplicateOrgRepair extends ToolBase {
} catch (Exception e) {
logger.error("Error getting orgs for user {}:{}", user.getName(), user.getId());
+ logger.error("Stack trace is: ", e);
}
}
} ).subscribeOn( scheduler ).toBlocking().lastOrDefault( null );
+
+ logger.info( "DuplicateOrgRepair augmented user orgs map");
}
@@ -216,10 +236,10 @@ public class DuplicateOrgRepair extends ToolBase {
args = new Object[] {
user.getName(), user.getId(), org.getName(), org.getId()};
logger.info( "Would remove user {}:{} org {}:{}", args);
- continue;
- }
- manager.addUserToOrg( user, bestOrg );
- manager.removeUserFromOrg( user, org );
+ } else {
+ manager.addUserToOrg( user, bestOrg );
+ manager.removeUserFromOrg( user, org );
+ }
}
Set<UUID> orgApps = new HashSet<UUID>( manager.getOrgApps( org ));
@@ -232,14 +252,16 @@ public class DuplicateOrgRepair extends ToolBase {
args = new Object[] {
appId, org.getName(), org.getId()};
logger.info( "Would remove app {} org {}:{}", args);
- continue;
- }
- manager.addAppToOrg( appId, bestOrg );
- manager.removeAppFromOrg( appId, org );
+ } else {
+ manager.addAppToOrg( appId, bestOrg );
+ manager.removeAppFromOrg( appId, org );
+ }
}
}
}
}
+
+ logger.info( "DuplicateOrgRepair merged duplicate orgs");
}
@@ -261,6 +283,7 @@ public class DuplicateOrgRepair extends ToolBase {
}
}
}
+ logger.info( "DuplicateOrgRepair renamed/removed duplicate orgs");
}
@@ -276,9 +299,11 @@ public class DuplicateOrgRepair extends ToolBase {
}
return oldest;
}
-
- class Manager implements DuplicateOrgInterface {
+
+
+
+ class RepairManager implements DuplicateOrgInterface {
private boolean dryRun = true;
@@ -291,6 +316,8 @@ public class DuplicateOrgRepair extends ToolBase {
public void call(Subscriber<? super Org> subscriber) {
subscriber.onStart();
try {
+ int count = 0;
+
Query query = new Query();
query.setLimit( MAX_ENTITY_FETCH );
query.setResultsLevel( Results.Level.ALL_PROPERTIES );
@@ -308,6 +335,10 @@ public class DuplicateOrgRepair extends ToolBase {
subscriber.onNext( org );
+ if ( count++ % 1000 == 0 ) {
+ logger.info("Emitted {} orgs", count );
+ }
+
// logger.info( "org: {}, \"{}\", {}", new Object[]{
// orgEntity.getProperty( "path" ),
// orgEntity.getUuid(),
@@ -337,6 +368,8 @@ public class DuplicateOrgRepair extends ToolBase {
public void call(Subscriber<? super OrgUser> subscriber) {
subscriber.onStart();
try {
+ int count = 0;
+
Query query = new Query();
query.setLimit( MAX_ENTITY_FETCH );
query.setResultsLevel( Results.Level.ALL_PROPERTIES );
@@ -352,7 +385,11 @@ public class DuplicateOrgRepair extends ToolBase {
orgUser.sourceValue = entity;
subscriber.onNext( orgUser );
-
+
+ if ( count++ % 1000 == 0 ) {
+ logger.info("Emitted {} users", count );
+ }
+
// logger.info( "org: {}, \"{}\", {}", new Object[]{
// entity.getProperty( "path" ),
// entity.getUuid(),
@@ -401,9 +438,7 @@ public class DuplicateOrgRepair extends ToolBase {
@Override
public void removeOrg(Org keeper, Org duplicate) throws Exception {
-
// we don't have a remove org API so rename org so that it is no longer a duplicate
-
OrganizationInfo orgInfo = managementService.getOrganizationByUuid( duplicate.getId() );
orgInfo.setName( "dup_" + keeper.getId() + "_" + RandomStringUtils.randomAlphanumeric(10) );
managementService.updateOrganization( orgInfo );
@@ -491,4 +526,29 @@ public class DuplicateOrgRepair extends ToolBase {
}
}
+
+
+ class DryRunManager extends RepairManager {
+
+ @Override
+ public void removeUserFromOrg(OrgUser user, Org org) throws Exception {
+ }
+
+ @Override
+ public void addUserToOrg(OrgUser user, Org org) throws Exception {
+ }
+
+ @Override
+ public void addAppToOrg(UUID appId, Org org) throws Exception {
+ }
+
+ @Override
+ public void removeAppFromOrg(UUID appId, Org org) throws Exception {
+ }
+
+ @Override
+ public void removeOrg(Org keeper, Org duplicate) throws Exception {
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de254f0a/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
index ac95748..4c92761 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/DuplicateOrgRepairTest.java
@@ -57,6 +57,8 @@ public class DuplicateOrgRepairTest {
int numOrgs = 10; // create 10 orgs and a dup for each
final DuplicateOrgRepair dor = new DuplicateOrgRepair();
+ dor.testing = true;
+
dor.manager = new Manager( numOrgs );
assertEquals( "must start with dups",
@@ -92,6 +94,7 @@ public class DuplicateOrgRepairTest {
int numOrgs = 10; // create 10 orgs and a dup for each
DuplicateOrgRepair dor = new DuplicateOrgRepair();
+ dor.testing = true;
dor.manager = new Manager( numOrgs );
assertEquals( "must start with dups",
@@ -131,6 +134,7 @@ public class DuplicateOrgRepairTest {
orgOwnerInfo1.getOwner(), orgOwnerInfo2.getOrganization(), false );
DuplicateOrgRepair dor = new DuplicateOrgRepair();
+ dor.testing = true;
dor.startTool( new String[] {}, false ); // false means do not call System.exit()
@@ -156,6 +160,7 @@ public class DuplicateOrgRepairTest {
"user_" + random2 + "@example.com", "password" );
DuplicateOrgRepair dor = new DuplicateOrgRepair();
+ dor.testing = true;
// start the tool so thaht Spring, Cassandra, etc/ gets initialized
dor.startTool( new String[] { "-dryrun", "true" }, false ); // false means do not call System.exit()
@@ -216,6 +221,7 @@ public class DuplicateOrgRepairTest {
orgOwnerInfo2.getOrganization().getUuid(), "app_" + RandomStringUtils.randomAlphanumeric( 10 ));
DuplicateOrgRepair dor = new DuplicateOrgRepair();
+ dor.testing = true;
// start the tool so that Spring, Cassandra, etc/ gets initialized
dor.startTool( new String[] { "-dryrun", "true" }, false ); // false means do not call System.exit()