You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/06/26 17:56:28 UTC
[1/2] incubator-usergrid git commit: Remove unnecessary code,
use user->org mapping to work around issues with
getOrganizationsForAdminUser().
Repository: incubator-usergrid
Updated Branches:
refs/heads/master 107a465e5 -> 5bdea9c0d
Remove unnecessary code, use user->org mapping to work around issues with getOrganizationsForAdminUser().
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4bd1115c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4bd1115c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4bd1115c
Branch: refs/heads/master
Commit: 4bd1115c443b3b215e882edec34e99b65735a69e
Parents: 107a465
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Jun 25 13:34:52 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Jun 25 13:34:52 2015 -0400
----------------------------------------------------------------------
.../org/apache/usergrid/tools/ExportAdmins.java | 268 ++++++++++---------
.../org/apache/usergrid/tools/ImportAdmins.java | 239 ++++++++---------
stack/tools/src/main/resources/log4j.properties | 6 +-
3 files changed, 252 insertions(+), 261 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4bd1115c/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index e781a1c..f5d1b1d 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@ -18,11 +18,14 @@
package org.apache.usergrid.tools;
+import au.com.bytecode.opencsv.CSVWriter;
import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.Results.Level;
import org.apache.usergrid.persistence.cassandra.CassandraService;
@@ -31,10 +34,12 @@ import org.codehaus.jackson.JsonGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.FileWriter;
import java.util.*;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
/**
@@ -48,20 +53,34 @@ public class ExportAdmins extends ExportingToolBase {
public static final String ADMIN_USERS_PREFIX = "admin-users";
public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";
private static final String READ_THREAD_COUNT = "readThreads";
+ private Map<String, List<Org>> orgMap = new HashMap<String, List<Org>>(80000);
private int readThreadCount;
-
-
+
+ AtomicInteger count = new AtomicInteger( 0 );
+
+
/**
* Represents an AdminUser that has been read and is ready for export.
*/
class AdminUserWriteTask {
Entity adminUser;
- Map<String, List<UUID>> collectionsByName;
- Map<String, List<ConnectionRef>> connectionsByType;
Map<String, Map<Object, Object>> dictionariesByName;
BiMap<UUID, String> orgNamesByUuid;
}
+
+ /**
+ * Represents an organization associated with a user.
+ */
+ private class Org {
+ UUID orgId;
+ String orgName;
+ public Org( UUID orgId, String orgName ) {
+ this.orgId = orgId;
+ this.orgName = orgName;
+ }
+ }
+
/**
* Export admin users using multiple threads.
@@ -93,6 +112,8 @@ public class ExportAdmins extends ExportingToolBase {
readThreadCount = 20;
}
+ buildOrgMap();
+
// start write queue worker
BlockingQueue<AdminUserWriteTask> writeQueue = new LinkedBlockingQueue<AdminUserWriteTask>();
@@ -124,7 +145,7 @@ public class ExportAdmins extends ExportingToolBase {
while (ids.size() > 0) {
for (UUID uuid : ids.getIds()) {
readQueue.add( uuid );
- logger.debug( "Added uuid to readQueue: " + uuid );
+ //logger.debug( "Added uuid to readQueue: " + uuid );
}
if (ids.getCursor() == null) {
break;
@@ -139,9 +160,15 @@ public class ExportAdmins extends ExportingToolBase {
}
logger.debug( "Waiting for write thread to complete" );
- writeThread.join();
+
+ boolean done = false;
+ while ( !done ) {
+ writeThread.join( 10000, 0 );
+ done = !writeThread.isAlive();
+ logger.info( "Wrote {} users", count.get() );
+ }
}
-
+
@Override
@SuppressWarnings("static-access")
@@ -157,6 +184,76 @@ public class ExportAdmins extends ExportingToolBase {
}
+ /**
+ * Shouldn't have to do this but getOrganizationsForAdminUser() is not 100% reliable in some Usergrid installations.
+ */
+ private void buildOrgMap() throws Exception {
+
+ logger.info("Building org map");
+
+ ExecutorService execService = Executors.newFixedThreadPool( this.readThreadCount );
+
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+ String queryString = "select *";
+ Query query = Query.fromQL( queryString );
+ query.withLimit( 1000 );
+ Results organizations = null;
+ int count = 0;
+ do {
+ organizations = em.searchCollection( em.getApplicationRef(), "groups", query );
+ for ( Entity organization : organizations.getEntities() ) {
+ execService.submit( new OrgMapWorker( organization ) );
+ }
+ count++;
+ if ( count % 1000 == 0 ) {
+ logger.info("Processed {} orgs for org map", count);
+ }
+ query.setCursor( organizations.getCursor() );
+ }
+ while ( organizations != null && organizations.hasCursor() );
+
+ execService.shutdown();
+ while ( !execService.awaitTermination( 10, TimeUnit.SECONDS ) ) {
+ logger.info("Processed {} orgs for map", orgMap.size() );
+ }
+ }
+
+
+ public class OrgMapWorker implements Runnable {
+ private final Entity orgEntity;
+
+ public OrgMapWorker( Entity orgEntity ) {
+ this.orgEntity = orgEntity;
+ }
+
+ @Override
+ public void run() {
+ try {
+ final String orgName = orgEntity.getProperty( "path" ).toString();
+ final UUID orgId = orgEntity.getUuid();
+ for (UserInfo user : managementService.getAdminUsersForOrganization( orgEntity.getUuid() )) {
+ try {
+ Entity admin = managementService.getAdminUserEntityByUuid( user.getUuid() );
+ List<Org> orgs = orgMap.get( admin.getProperty( "username" ) );
+ if (orgs == null) {
+ orgs = new ArrayList<Org>();
+ orgMap.put( admin.getProperty( "username" ).toString().toLowerCase(), orgs );
+ }
+ orgs.add( new Org( orgId, orgName ) );
+
+ //logger.debug("Added org {} for user {}", orgName, admin.getProperty( "username" ));
+
+ } catch (Exception e) {
+ logger.warn( "Cannot get orgs for userId {}", user.getUuid() );
+ }
+ }
+ } catch ( Exception e ) {
+ logger.error("Error getting users for org {}:{}", orgEntity.getName(), orgEntity.getUuid());
+ }
+ }
+ }
+
+
public class AdminUserReader implements Runnable {
private boolean done = true;
@@ -189,7 +286,7 @@ public class ExportAdmins extends ExportingToolBase {
UUID uuid = null;
try {
uuid = readQueue.poll( 30, TimeUnit.SECONDS );
- logger.debug("Got item from entityId queue: " + uuid );
+ //logger.debug("Got item from entityId queue: " + uuid );
if ( uuid == null && done ) {
break;
@@ -200,10 +297,8 @@ public class ExportAdmins extends ExportingToolBase {
AdminUserWriteTask task = new AdminUserWriteTask();
task.adminUser = entity;
- addCollectionsToTask( task, entity );
addDictionariesToTask( task, entity );
- addConnectionsToTask( task, entity );
- addOrganizationsToTask( task, entity );
+ addOrganizationsToTask( task );
writeQueue.add( task );
@@ -214,41 +309,14 @@ public class ExportAdmins extends ExportingToolBase {
}
- private void addCollectionsToTask(AdminUserWriteTask task, Entity entity) throws Exception {
-
- EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
- Set<String> collections = em.getCollections( entity );
- if ((collections == null) || collections.isEmpty()) {
- return;
- }
-
- task.collectionsByName = new HashMap<String, List<UUID>>();
-
- for (String collectionName : collections) {
-
- List<UUID> uuids = task.collectionsByName.get( collectionName );
- if ( uuids == null ) {
- uuids = new ArrayList<UUID>();
- task.collectionsByName.put( collectionName, uuids );
- }
-
- Results collectionMembers = em.getCollection( entity, collectionName, null, 100000, Level.IDS, false );
-
- List<UUID> entityIds = collectionMembers.getIds();
-
- if ((entityIds != null) && !entityIds.isEmpty()) {
- for (UUID childEntityUUID : entityIds) {
- uuids.add( childEntityUUID );
- }
- }
- }
- }
-
-
private void addDictionariesToTask(AdminUserWriteTask task, Entity entity) throws Exception {
EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
Set<String> dictionaries = em.getDictionaries( entity );
+
+ if ( dictionaries.isEmpty() ) {
+ logger.error("User {}:{} has no dictionaries", task.adminUser.getName(), task.adminUser.getUuid() );
+ }
task.dictionariesByName = new HashMap<String, Map<Object, Object>>();
@@ -258,32 +326,26 @@ public class ExportAdmins extends ExportingToolBase {
}
}
+ private void addOrganizationsToTask(AdminUserWriteTask task) throws Exception {
- private void addConnectionsToTask(AdminUserWriteTask task, Entity entity) throws Exception {
- EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
-
- task.connectionsByType = new HashMap<String, List<ConnectionRef>>();
-
- Set<String> connectionTypes = em.getConnectionTypes( entity );
- for (String connectionType : connectionTypes) {
+ task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() );
- List<ConnectionRef> connRefs = task.connectionsByType.get( connectionType );
- if ( connRefs == null ) {
- connRefs = new ArrayList<ConnectionRef>();
- }
-
- Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Level.IDS );
- List<ConnectionRef> connections = results.getConnections();
-
- for (ConnectionRef connectionRef : connections) {
- connRefs.add( connectionRef );
+ List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
+
+ if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {
+ BiMap<UUID, String> bimap = HashBiMap.create();
+ for (Org org : orgs) {
+ bimap.put( org.orgId, org.orgName );
}
+ task.orgNamesByUuid = bimap;
+ }
+
+ if ( task.orgNamesByUuid.isEmpty() ) {
+ logger.error("{}:{}:{} has no orgs", new Object[] {
+ task.adminUser.getProperty("username"),
+ task.adminUser.getProperty("email"),
+ task.adminUser.getUuid() } );
}
- }
-
-
- private void addOrganizationsToTask(AdminUserWriteTask task, Entity entity) throws Exception {
- task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( entity.getUuid() );
}
public void setDone(boolean done) {
@@ -325,8 +387,6 @@ public class ExportAdmins extends ExportingToolBase {
getJsonGenerator( createOutputFile( ADMIN_USER_METADATA_PREFIX, em.getApplication().getName() ) );
metadataFile.writeStartObject();
- int count = 0;
-
while ( true ) {
try {
@@ -340,18 +400,20 @@ public class ExportAdmins extends ExportingToolBase {
echo( task.adminUser );
// write metadata to metadata file
- saveCollections( metadataFile, task );
- saveConnections( metadataFile, task );
- saveOrganizations( metadataFile, task );
- saveDictionaries( metadataFile, task );
+ metadataFile.writeFieldName( task.adminUser.getUuid().toString() );
+ metadataFile.writeStartObject();
- logger.debug("Exported user {}", task.adminUser.getProperty( "email" ));
-
- count++;
- if ( count % 1000 == 0 ) {
- logger.info("Exported {} admin users", count);
- }
+ saveOrganizations( metadataFile, task );
+ saveDictionaries( metadataFile, task );
+
+ metadataFile.writeEndObject();
+
+ logger.debug( "Exported user {}:{}:{}", new Object[] {
+ task.adminUser.getProperty("username"),
+ task.adminUser.getProperty("email"),
+ task.adminUser.getUuid() } );
+ count.addAndGet( 1 );
} catch (InterruptedException e) {
throw new Exception("Interrupted", e);
@@ -364,32 +426,7 @@ public class ExportAdmins extends ExportingToolBase {
usersFile.writeEndArray();
usersFile.close();
- logger.info("Exported TOTAL {} admin users", count);
- }
-
-
- private void saveCollections( JsonGenerator jg, AdminUserWriteTask task ) throws Exception {
-
- jg.writeFieldName( task.adminUser.getUuid().toString() );
- jg.writeStartObject();
-
- for (String collectionName : task.collectionsByName.keySet() ) {
-
- jg.writeFieldName( collectionName );
- jg.writeStartArray();
-
- List<UUID> entityIds = task.collectionsByName.get( collectionName );
-
- if ((entityIds != null) && !entityIds.isEmpty()) {
- for (UUID childEntityUUID : entityIds) {
- jg.writeObject( childEntityUUID.toString() );
- }
- }
-
- jg.writeEndArray();
- }
-
- jg.writeEndObject();
+ logger.info( "Exported TOTAL {} admin users", count );
}
@@ -421,27 +458,6 @@ public class ExportAdmins extends ExportingToolBase {
}
- private void saveConnections( JsonGenerator jg, AdminUserWriteTask task ) throws Exception {
-
- jg.writeFieldName( "connections" );
- jg.writeStartObject();
-
- for (String connectionType : task.connectionsByType.keySet() ) {
-
- jg.writeFieldName( connectionType );
- jg.writeStartArray();
-
- List<ConnectionRef> connections = task.connectionsByType.get( connectionType );
- for (ConnectionRef connectionRef : connections) {
- jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
- }
-
- jg.writeEndArray();
- }
- jg.writeEndObject();
- }
-
-
private void saveOrganizations( JsonGenerator jg, AdminUserWriteTask task ) throws Exception {
final BiMap<UUID, String> orgs = task.orgNamesByUuid;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4bd1115c/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
index d05c9a8..857e97e 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
@@ -40,6 +40,7 @@ import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
@@ -47,9 +48,11 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEM
/**
- * TODO: REFACTOR EVERYTHING TO USE JSON NODES
- * Example on how to run:
- * java -jar usergrid-tools.jar ImportAdmins -host cassandraHost -v -inputDir exportFilesDirectory
+ * Usage example:
+ *
+ * java -Xmx8000m -Dlog4j.configuration=file:/home/dmjohnson/import-admins/log4j.properties -classpath . \
+ * -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 -host cca03eadn \
+ * -inputDir=/home/dmjohnson/export-admins/exported > import3.log 2>&1 &
*/
public class ImportAdmins extends ToolBase {
@@ -73,6 +76,9 @@ public class ImportAdmins extends ToolBase {
JsonFactory jsonFactory = new JsonFactory();
+ AtomicInteger userCount = new AtomicInteger( 0 );
+ AtomicInteger metadataCount = new AtomicInteger( 0 );
+
@Override
@SuppressWarnings("static-access")
@@ -102,8 +108,8 @@ public class ImportAdmins extends ToolBase {
options.addOption(hostOption);
options.addOption(writeThreads);
options.addOption(auditThreads);
- options.addOption(inputDir);
- options.addOption(verbose);
+ options.addOption( inputDir );
+ options.addOption( verbose );
return options;
}
@@ -126,16 +132,12 @@ public class ImportAdmins extends ToolBase {
}
if (line.hasOption(WRITE_THREAD_COUNT)) {
- writeThreadCount = Integer.parseInt(line.getOptionValue(WRITE_THREAD_COUNT));
+ writeThreadCount = Integer.parseInt( line.getOptionValue(WRITE_THREAD_COUNT));
}
importAdminUsers(writeThreadCount, auditThreadCount);
- importMetadata(writeThreadCount);
-
- // forces the counters to flush
-// logger.info( "Sleeping 35 seconds for batcher" );
-// Thread.sleep( 35000 );
+ importMetadata( writeThreadCount );
}
@@ -190,10 +192,11 @@ public class ImportAdmins extends ToolBase {
@SuppressWarnings("unchecked")
Map<String, Object> entityProps = jp.readValueAs(HashMap.class);
- if (loopCounter % 100 == 1)
- logger.info("Publishing to queue... counter=" + loopCounter);
+ if (loopCounter % 1000 == 0) {
+ logger.debug( "Publishing to queue... counter=" + loopCounter );
+ }
- workQueue.add(entityProps);
+ workQueue.add( entityProps );
}
waitForQueueAndMeasure(workQueue, adminWriteThreads, "Admin Write");
@@ -209,15 +212,15 @@ public class ImportAdmins extends ToolBase {
final Map<Stoppable, Thread> threadMap,
final String identifier) throws InterruptedException {
double rateAverageSum = 0;
- int iterationCounter = 0;
+ int iterations = 0;
while (!workQueue.isEmpty()) {
- iterationCounter += 1;
+ iterations += 1;
int sizeLast = workQueue.size();
long lastTime = System.currentTimeMillis();
logger.info("Queue {} is not empty, remaining size={}, waiting...", identifier, sizeLast);
- Thread.sleep(5000);
+ Thread.sleep(10000);
long timeNow = System.currentTimeMillis();
int sizeNow = workQueue.size();
@@ -229,12 +232,11 @@ public class ImportAdmins extends ToolBase {
double rateLast = (double) processed / (timeDelta / 1000);
rateAverageSum += rateLast;
- long timeRemaining = (long) ( sizeLast / (rateAverageSum / iterationCounter) );
+ long timeRemaining = (long) ( sizeLast / (rateAverageSum / iterations) );
- logger.info(
- String.format("++PROGRESS (%s): sizeLast=%s nowSize=%s processed=%s rateLast=%s/s rateAvg=%s/s timeRemaining=%s(s)",
- identifier, sizeLast, sizeNow, processed, rateLast, (rateAverageSum / iterationCounter), timeRemaining)
- );
+ logger.info("++PROGRESS ({}): sizeLast={} nowSize={} processed={} rateLast={}/s rateAvg={}/s timeRemaining={}s",
+ new Object[] {
+ identifier, sizeLast, sizeNow, processed, rateLast, (rateAverageSum / iterations), timeRemaining } );
}
for (Stoppable worker : threadMap.keySet()) {
@@ -249,6 +251,7 @@ public class ImportAdmins extends ToolBase {
workerThread.start();
adminAuditThreads.put(worker, workerThread);
}
+ logger.info("Started {} admin auditors", workerCount);
}
@@ -263,6 +266,8 @@ public class ImportAdmins extends ToolBase {
workerThread.start();
adminWriteThreads.put(worker, workerThread);
}
+
+ logger.info("Started {} admin workers", workerCount);
}
@@ -284,8 +289,8 @@ public class ImportAdmins extends ToolBase {
private JsonParser getJsonParserForFile(File organizationFile) throws Exception {
- JsonParser jp = jsonFactory.createJsonParser(organizationFile);
- jp.setCodec(new ObjectMapper());
+ JsonParser jp = jsonFactory.createJsonParser( organizationFile );
+ jp.setCodec( new ObjectMapper() );
return jp;
}
@@ -312,10 +317,12 @@ public class ImportAdmins extends ToolBase {
for (int x = 0; x < writeThreadCount; x++) {
ImportMetadataWorker worker = new ImportMetadataWorker(workQueue);
- Thread workerThread = new Thread(worker, "ImportMetadataTask-" + x);
+ Thread workerThread = new Thread(worker, "ImportMetadataTask-" + x );
workerThread.start();
metadataWorkerThreadMap.put(worker, workerThread);
}
+
+ logger.info( "Started {} metadata workers", writeThreadCount );
}
@@ -362,7 +369,6 @@ public class ImportAdmins extends ToolBase {
Map<String, Object> metadata = (Map<String, Object>) jp.readValueAs(Map.class);
workQueue.put(new ImportMetadataTask(entityRef, metadata));
-// importEntityMetadata(em, entityRef, metadata);
}
}
@@ -380,82 +386,11 @@ public class ImportAdmins extends ToolBase {
private void importEntityMetadata(
EntityManager em, EntityRef entityRef, Map<String, Object> metadata) throws Exception {
- Map<String, Object> connectionsMap = (Map<String, Object>) metadata.get("connections");
-
- if (connectionsMap != null && !connectionsMap.isEmpty()) {
- for (String type : connectionsMap.keySet()) {
- try {
- UUID uuid = UUID.fromString((String) connectionsMap.get(type));
- EntityRef connectedEntityRef = em.getRef(uuid);
- em.createConnection(entityRef, type, connectedEntityRef);
-
- logger.debug("Creating connection from {} type {} target {}",
- new Object[]{entityRef, type, connectedEntityRef});
-
- } catch (Exception e) {
- if (logger.isDebugEnabled()) {
- logger.error("Error importing connection of type "
- + type + " for user " + entityRef.getUuid(), e);
- } else {
- logger.error("Error importing connection of type "
- + type + " for user " + entityRef.getUuid());
- }
- }
- }
- }
-
- Map<String, Object> dictionariesMap = (Map<String, Object>) metadata.get("dictionaries");
-
- if (dictionariesMap != null && !dictionariesMap.isEmpty()) {
- for (String name : dictionariesMap.keySet()) {
- try {
- Map<String, Object> dictionary = (Map<String, Object>) dictionariesMap.get(name);
- em.addMapToDictionary(entityRef, name, dictionary);
-
- logger.debug("Creating dictionary for {} name {} map {}",
- new Object[]{entityRef, name, dictionary});
-
- } catch (Exception e) {
- if (logger.isDebugEnabled()) {
- logger.error("Error importing dictionary name "
- + name + " for user " + entityRef.getUuid(), e);
- } else {
- logger.error("Error importing dictionary name "
- + name + " for user " + entityRef.getUuid());
- }
- }
- }
- }
-
- List<String> collectionsList = (List<String>) metadata.get("collections");
- if (collectionsList != null && !collectionsList.isEmpty()) {
- for (String name : collectionsList) {
- try {
- UUID uuid = UUID.fromString((String) connectionsMap.get(name));
- EntityRef collectedEntityRef = em.getRef(uuid);
- em.addToCollection(entityRef, name, collectedEntityRef);
-
- logger.debug("Add to collection of {} name {} entity {}",
- new Object[]{entityRef, name, collectedEntityRef});
-
- } catch (Exception e) {
- if (logger.isDebugEnabled()) {
- logger.error("Error adding to collection "
- + name + " for user " + entityRef.getUuid(), e);
- } else {
- logger.error("Error adding to collection "
- + name + " for user " + entityRef.getUuid());
- }
- }
- }
- }
-
-
List<Object> organizationsList = (List<Object>) metadata.get("organizations");
if (organizationsList != null && !organizationsList.isEmpty()) {
User user = em.get(entityRef, User.class);
-
+
if (user == null) {
logger.error("User with uuid={} not found, not adding to organizations");
@@ -476,19 +411,55 @@ public class ImportAdmins extends ToolBase {
managementService.createOrganization(orgUuid, orgName, userInfo, false);
orgInfo = managementService.getOrganizationByUuid(orgUuid);
- logger.debug("Created new org {} for user {}",
- new Object[]{orgInfo.getName(), user.getEmail()});
+ logger.debug( "Created new org {} for user {}",
+ new Object[]{orgInfo.getName(), user.getEmail()} );
} catch (DuplicateUniquePropertyExistsException dpee) {
- logger.error("Org {} already exists", orgName);
+ logger.debug( "Org {} already exists", orgName );
+ }
+ } else {
+ try {
+ managementService.addAdminUserToOrganization( userInfo, orgInfo, false );
+ logger.debug( "Added user {} to org {}", new Object[]{user.getEmail(), orgName} );
+
+ } catch ( Exception e ) {
+ logger.error( "Error Adding user {} to org {}", new Object[]{user.getEmail(), orgName} );
}
+ }
+ }
+ }
+
+ } else {
+ logger.warn("User {} has no organizations", entityRef.getUuid() );
+ }
+
+ Map<String, Object> dictionariesMap = (Map<String, Object>) metadata.get("dictionaries");
+
+ if (dictionariesMap != null && !dictionariesMap.isEmpty()) {
+ for (String name : dictionariesMap.keySet()) {
+ try {
+ Map<String, Object> dictionary = (Map<String, Object>) dictionariesMap.get(name);
+ em.addMapToDictionary( entityRef, name, dictionary);
+
+ logger.debug( "Creating dictionary for {} name {}",
+ new Object[]{entityRef, name} );
+
+ } catch (Exception e) {
+ if (logger.isDebugEnabled()) {
+ logger.error("Error importing dictionary name "
+ + name + " for user " + entityRef.getUuid(), e);
} else {
- managementService.addAdminUserToOrganization(userInfo, orgInfo, false);
- logger.debug("Added user {} to org {}", new Object[]{user.getEmail(), orgName});
+ logger.error("Error importing dictionary name "
+ + name + " for user " + entityRef.getUuid());
}
}
}
+
+ } else {
+ logger.warn("User {} has no dictionaries", entityRef.getUuid() );
}
+
+
}
@@ -552,8 +523,7 @@ public class ImportAdmins extends ToolBase {
String type = getType(entityProps);
if (em.get(uuid) == null) {
- logger.error("Holy hell, we wrote an entity and it's missing. " +
- "Entity Id was {} and type is {}", uuid, type);
+ logger.error( "FATAL ERROR: wrote an entity {}:{} and it's missing", uuid, type );
System.exit(1);
}
@@ -563,8 +533,12 @@ public class ImportAdmins extends ToolBase {
long duration = stopTime - startTime;
durationSum += duration;
- logger.debug(String.format("Audited [%s]th admin", count));
- logger.info(String.format("Average Audit Rate: %s(ms)", durationSum / count));
+
+ //logger.debug( "Audited {}th admin", count );
+
+ if ( count % 100 == 0 ) {
+ logger.info( "Audited {}. Average Audit Rate: {}(ms)", count, durationSum / count );
+ }
} catch (InterruptedException e) {
e.printStackTrace();
@@ -572,8 +546,6 @@ public class ImportAdmins extends ToolBase {
e.printStackTrace();
}
}
-
- logger.warn("Done!");
}
}
@@ -619,24 +591,28 @@ public class ImportAdmins extends ToolBase {
Thread.sleep(1000);
continue;
}
-
- count++;
+
long startTime = System.currentTimeMillis();
+
importEntityMetadata(em, task.entityRef, task.metadata);
+
+ metadataCount.addAndGet( 1 );
long stopTime = System.currentTimeMillis();
-
long duration = stopTime - startTime;
durationSum += duration;
- logger.debug(String.format("Imported [%s]th metadata", count));
- logger.info(String.format("Average metadata Imported Rate: %s(ms)", durationSum / count));
+ count++;
+
+ //logger.debug( "Imported {}th metadata", count );
+
+ if ( count % 30 == 0 ) {
+ logger.info( "Imported {} metadata of total {}. Average metadata Imported Rate: {}(ms)",
+ new Object[] { count, metadataCount.get(), durationSum / count });
+ }
} catch (Exception e) {
- e.printStackTrace();
- logger.debug("EXCEPTION", e);
+ logger.debug("Error reading writing metadata", e);
}
}
-
- logger.warn("Done!");
}
}
@@ -650,7 +626,6 @@ public class ImportAdmins extends ToolBase {
public ImportAdminWorker(final BlockingQueue<Map<String, Object>> workQueue,
final BlockingQueue<Map<String, Object>> auditQueue) {
- logger.info("New Worker!");
this.workQueue = workQueue;
this.auditQueue = auditQueue;
}
@@ -684,27 +659,29 @@ public class ImportAdmins extends ToolBase {
UUID uuid = getId(entityProps);
String type = getType(entityProps);
-
try {
long startTime = System.currentTimeMillis();
+
em.create(uuid, type, entityProps);
+
+ logger.debug( "Imported admin user {} / {}",
+ new Object[] { uuid, entityProps.get( "username" ) } );
+
+ userCount.addAndGet( 1 );
auditQueue.put(entityProps);
long stopTime = System.currentTimeMillis();
-
long duration = stopTime - startTime;
durationSum += duration;
-
+
count++;
- logger.debug(String.format("Imported [%s]th admin user %s / %s", count, uuid, entityProps.get("username")));
- logger.info(String.format("Average Creation Rate: %s(ms)", durationSum / count));
-
- if (count % 100 == 0) {
- logger.info("Imported {} admin users", count);
+ if (count % 30 == 0) {
+ logger.info( "Imported {} admin users of total {}. Average Creation Rate: {}ms",
+ new Object[] { count, userCount.get(), durationSum / count });
}
+
} catch (DuplicateUniquePropertyExistsException de) {
- logger.warn("Unable to create entity. It appears to be a duplicate: " +
- "id={}, type={}, name={}, username={}",
- new Object[]{uuid, type, entityProps.get("name"), entityProps.get("username")});
+ logger.warn("Unable to create admin user {}:{}, duplicate property {}",
+ new Object[]{ uuid, entityProps.get("username"), de.getPropertyName() });
if (logger.isDebugEnabled()) {
logger.debug("Exception", de);
}
@@ -716,8 +693,6 @@ public class ImportAdmins extends ToolBase {
}
}
-
- logger.warn("Done!");
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4bd1115c/stack/tools/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/resources/log4j.properties b/stack/tools/src/main/resources/log4j.properties
index 73ade48..6cf0a92 100644
--- a/stack/tools/src/main/resources/log4j.properties
+++ b/stack/tools/src/main/resources/log4j.properties
@@ -18,7 +18,7 @@
# and the pattern to %c instead of %l. (%l is slower.)
# output messages into a rolling log file as well as stdout
-log4j.rootLogger=INFO,stdout
+log4j.rootLogger=WARN,stdout
# stdout
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
@@ -26,9 +26,9 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
-log4j.logger.org.apache.usergrid.tools=DEBUG
-log4j.logger.org.apache.usergrid.management.cassandra=DEBUB
+log4j.logger.org.apache.usergrid.tools=INFO
+log4j.logger.org.apache.usergrid.management.cassandra=WARN
log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN
log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN
log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN
[2/2] incubator-usergrid git commit: Minor tweaks to comments.
Posted by gr...@apache.org.
Minor tweaks to comments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5bdea9c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5bdea9c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5bdea9c0
Branch: refs/heads/master
Commit: 5bdea9c0d2ba7642dd94205779abb9ac8650e1b8
Parents: 4bd1115
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Jun 25 16:16:01 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Jun 25 16:16:01 2015 -0400
----------------------------------------------------------------------
.../org/apache/usergrid/tools/ExportAdmins.java | 25 +++++++++++++++-----
.../org/apache/usergrid/tools/ImportAdmins.java | 21 ++++++++++++----
2 files changed, 35 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5bdea9c0/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index f5d1b1d..d3d6371 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@ -14,11 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.usergrid.tools;
-import au.com.bytecode.opencsv.CSVWriter;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import org.apache.commons.cli.CommandLine;
@@ -26,7 +24,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.usergrid.management.UserInfo;
-import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.Results.Level;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.utils.StringUtils;
@@ -34,7 +35,6 @@ import org.codehaus.jackson.JsonGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.FileWriter;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -43,9 +43,22 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEM
/**
- * Export Admin Users and metadata including organizations.
+ * Export Admin Users and metadata including organizations and passwords.
*
- * java -jar usergrid-tools.jar ExportAdmins
+ * Usage Example:
+ *
+ * java -Xmx8000m -Dlog4j.configuration=file:/home/me/log4j.properties -classpath . \
+ * -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 \
+ * -host casshost -inputDir=/home/me/export-data
+ *
+ * If you want to provide any property overrides, put properties file named usergrid-custom-tools.properties
+ * in the same directory where you run the above command. For example, you might want to set the Cassandra
+ * client threads and export from a specific set of keyspaces:
+ *
+ * cassandra.connections=110
+ * cassandra.system.keyspace=My_Usergrid
+ * cassandra.application.keyspace=My_Usergrid_Applications
+ * cassandra.lock.keyspace=My_Usergrid_Locks
*/
public class ExportAdmins extends ExportingToolBase {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5bdea9c0/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
index 857e97e..39384e6 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
@@ -48,11 +48,22 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEM
/**
- * Usage example:
+ * Import Admin Users and metadata including organizations and passwords.
*
- * java -Xmx8000m -Dlog4j.configuration=file:/home/dmjohnson/import-admins/log4j.properties -classpath . \
- * -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 -host cca03eadn \
- * -inputDir=/home/dmjohnson/export-admins/exported > import3.log 2>&1 &
+ * Usage Example:
+ *
+ * java -Xmx8000m -Dlog4j.configuration=file:/home/me/log4j.properties -classpath . \
+ * -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 \
+ * -host casshost -inputDir=/home/me/import-data
+ *
+ * If you want to provide any property overrides, put properties file named usergrid-custom-tools.properties
+ * in the same directory where you run the above command. For example, you might want to set the Cassandra
+ * client threads and import to a specific set of keyspaces:
+ *
+ * cassandra.connections=110
+ * cassandra.system.keyspace=My_Other_Usergrid
+ * cassandra.application.keyspace=My_Other_Usergrid_Applications
+ * cassandra.lock.keyspace=My_Other_Usergrid_Locks
*/
public class ImportAdmins extends ToolBase {
@@ -651,7 +662,7 @@ public class ImportAdmins extends ToolBase {
if (entityProps == null) {
logger.warn("Reading from admin import queue was null!");
- Thread.sleep(1000);
+ Thread.sleep( 1000 );
continue;
}