You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/06/19 16:42:23 UTC
[01/13] incubator-usergrid git commit: Added threading and queueing
to imports.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o e1cdc7368 -> 81082f358
Added threading and queueing to imports.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c0b89db5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c0b89db5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c0b89db5
Branch: refs/heads/two-dot-o
Commit: c0b89db57ae43ffb1b94780ea79c314b38de5a60
Parents: 5fbe31f
Author: Jeff West <jw...@apigee.com>
Authored: Thu Jun 11 07:47:39 2015 -0700
Committer: Jeff West <jw...@apigee.com>
Committed: Thu Jun 11 07:47:39 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/tools/ImportAdmins.java | 598 ++++++++++++++-----
1 file changed, 450 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0b89db5/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
index 49757ee..7d49459 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
@@ -36,10 +36,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
@@ -53,55 +53,85 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEM
*/
public class ImportAdmins extends ToolBase {
- private static final Logger logger = LoggerFactory.getLogger( ImportAdmins.class );
+ private static final Logger logger = LoggerFactory.getLogger(ImportAdmins.class);
/**
* Input directory where the .json export files are
*/
static final String INPUT_DIR = "inputDir";
+ static final String WRITE_THREAD_COUNT = "writeThreads";
+ static final String AUDIT_THREAD_COUNT = "auditThreads";
static File importDir;
static final String DEFAULT_INPUT_DIR = "export";
+ private Map<Stoppable, Thread> adminWriteThreads = new HashMap<Stoppable, Thread>();
+ private Map<Stoppable, Thread> adminAuditThreads = new HashMap<Stoppable, Thread>();
+ private Map<Stoppable, Thread> metadataWorkerThreadMap = new HashMap<Stoppable, Thread>();
+
+
JsonFactory jsonFactory = new JsonFactory();
@Override
- @SuppressWarnings( "static-access" )
+ @SuppressWarnings("static-access")
public Options createOptions() {
- Option hostOption = OptionBuilder.withArgName( "host" )
- .hasArg().withDescription( "Cassandra host" ).create( "host" );
+ Option hostOption = OptionBuilder.withArgName("host")
+ .hasArg()
+ .withDescription("Cassandra host").create("host");
Option inputDir = OptionBuilder
- .hasArg().withDescription( "input directory -inputDir" ).create( INPUT_DIR );
+ .hasArg()
+ .withDescription("input directory -inputDir").create(INPUT_DIR);
+
+ Option writeThreads = OptionBuilder
+ .hasArg()
+ .withDescription("Write Threads -writeThreads").create(WRITE_THREAD_COUNT);
+
+ Option auditThreads = OptionBuilder
+ .hasArg()
+ .withDescription("Audit Threads -auditThreads").create(AUDIT_THREAD_COUNT);
Option verbose = OptionBuilder
- .withDescription( "Print on the console an echo of the content written to the file" )
- .create( VERBOSE );
+ .withDescription("Print on the console an echo of the content written to the file")
+ .create(VERBOSE);
Options options = new Options();
- options.addOption( hostOption );
- options.addOption( inputDir );
- options.addOption( verbose );
+ options.addOption(hostOption);
+ options.addOption(writeThreads);
+ options.addOption(auditThreads);
+ options.addOption(inputDir);
+ options.addOption(verbose);
return options;
}
@Override
- public void runTool( CommandLine line ) throws Exception {
+ public void runTool(CommandLine line) throws Exception {
startSpring();
- setVerbose( line );
+ setVerbose(line);
+
+ openImportDirectory(line);
- openImportDirectory( line );
+ int auditThreadCount = 1;
+ int writeThreadCount = 1;
+
+ if (line.hasOption(AUDIT_THREAD_COUNT)) {
+ auditThreadCount = Integer.parseInt(line.getOptionValue(AUDIT_THREAD_COUNT));
+ }
+
+ if (line.hasOption(WRITE_THREAD_COUNT)) {
+ writeThreadCount = Integer.parseInt(line.getOptionValue(WRITE_THREAD_COUNT));
+ }
- importAdminUsers();
+ importAdminUsers(writeThreadCount, auditThreadCount);
- importMetadata();
+ importMetadata(writeThreadCount);
// forces the counters to flush
// logger.info( "Sleeping 35 seconds for batcher" );
@@ -112,17 +142,17 @@ public class ImportAdmins extends ToolBase {
/**
* Import admin users.
*/
- private void importAdminUsers() throws Exception {
- String[] fileNames = importDir.list( new PrefixFileFilter( ExportAdmins.ADMIN_USERS_PREFIX + "." ) );
- logger.info( "Applications to read: " + fileNames.length );
+ private void importAdminUsers(int writeThreadCount, int auditThreadCount) throws Exception {
- //this fails on the second run of the applications find out why.
- for ( String fileName : fileNames ) {
+ String[] fileNames = importDir.list(new PrefixFileFilter(ExportAdmins.ADMIN_USERS_PREFIX + "."));
+
+ logger.info("Applications to read: " + fileNames.length);
+
+ for (String fileName : fileNames) {
try {
- importAdminUsers( fileName );
- }
- catch ( Exception e ) {
- logger.warn( "Unable to import application: " + fileName, e );
+ importAdminUsers(fileName, writeThreadCount, auditThreadCount);
+ } catch (Exception e) {
+ logger.warn("Unable to import application: " + fileName, e);
}
}
}
@@ -133,84 +163,129 @@ public class ImportAdmins extends ToolBase {
*
* @param fileName Name of admin user data file.
*/
- private void importAdminUsers( String fileName ) throws Exception {
+ private void importAdminUsers(final String fileName,
+ final int writeThreadCount,
+ final int auditThreadCount) throws Exception {
int count = 0;
- File adminUsersFile = new File( importDir, fileName );
+ File adminUsersFile = new File(importDir, fileName);
+
+ logger.info("----- Loading file: " + adminUsersFile.getAbsolutePath());
+ JsonParser jp = getJsonParserForFile(adminUsersFile);
+
+ int loopCounter = 0;
- logger.info( "----- Loading file: " + adminUsersFile.getAbsolutePath() );
- JsonParser jp = getJsonParserForFile( adminUsersFile );
+ BlockingQueue<Map<String, Object>> workQueue = new LinkedBlockingQueue<Map<String, Object>>();
+ BlockingQueue<Map<String, Object>> auditQueue = new LinkedBlockingQueue<Map<String, Object>>();
+
+ startAdminWorkers(workQueue, auditQueue, writeThreadCount);
+ startAdminAuditors(auditQueue, auditThreadCount);
JsonToken token = jp.nextToken();
- validateStartArray( token );
+ validateStartArray(token);
- EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+ while (jp.nextValue() != JsonToken.END_ARRAY) {
+ loopCounter += 1;
- while ( jp.nextValue() != JsonToken.END_ARRAY ) {
+ @SuppressWarnings("unchecked")
+ Map<String, Object> entityProps = jp.readValueAs(HashMap.class);
+ if (loopCounter % 100 == 1)
+ logger.info("Publishing to queue... counter=" + loopCounter);
- @SuppressWarnings( "unchecked" )
- Map<String, Object> entityProps = jp.readValueAs( HashMap.class );
+ workQueue.add(entityProps);
+ }
- // Import/create the entity
- UUID uuid = getId( entityProps );
- String type = getType( entityProps );
+ waitForQueueAndMeasure(workQueue, adminWriteThreads, "Admin Write");
+ waitForQueueAndMeasure(auditQueue, adminAuditThreads, "Admin Audit");
+ logger.info("----- End: Imported {} admin users from file {}",
+ count, adminUsersFile.getAbsolutePath());
- try {
- em.create( uuid, type, entityProps );
+ jp.close();
+ }
- logger.debug( "Imported admin user {} {}", uuid, entityProps.get( "username" ) );
- count++;
- if ( count % 1000 == 0 ) {
- logger.info("Imported {} admin users", count);
- }
- }
- catch ( DuplicateUniquePropertyExistsException de ) {
- logger.warn( "Unable to create entity. It appears to be a duplicate: " +
- "id={}, type={}, name={}, username={}",
- new Object[] { uuid, type, entityProps.get("name"), entityProps.get("username")});
- if ( logger.isDebugEnabled() ) {
- logger.debug( "Exception" , de );
- }
- continue;
- }
+ private static void waitForQueueAndMeasure(final BlockingQueue workQueue,
+ final Map<Stoppable, Thread> threadMap,
+ final String identifier) throws InterruptedException {
+ double rateAverageSum = 0;
+ int iterationCounter = 0;
- if ( em.get( uuid ) == null ) {
- logger.error( "Holy hell, we wrote an entity and it's missing. " +
- "Entity Id was {} and type is {}", uuid, type );
- System.exit( 1 );
- }
- echo( entityProps );
+ while (!workQueue.isEmpty()) {
+ iterationCounter += 1;
+
+ int sizeLast = workQueue.size();
+ long lastTime = System.currentTimeMillis();
+ logger.info("Queue {} is not empty, remaining size={}, waiting...", identifier, sizeLast);
+ Thread.sleep(5000);
+
+ long timeNow = System.currentTimeMillis();
+ int sizeNow = workQueue.size();
+
+ int processed = sizeLast - sizeNow;
+
+ long timeDelta = timeNow - lastTime;
+
+ double rateLast = (double) processed / (timeDelta / 1000);
+ rateAverageSum += rateLast;
+
+ long timeRemaining = sizeLast / (long) (rateAverageSum / iterationCounter);
+
+ logger.info(
+ String.format("++PROGRESS (%s): sizeLast=%s nowSize=%s processed=%s rateLast=%s/s rateAvg=%s/s timeRemaining=%s(s)",
+ identifier, sizeLast, sizeNow, processed, rateLast, (rateAverageSum / iterationCounter), timeRemaining)
+ );
}
- logger.info( "----- End: Imported {} admin users from file {}",
- count, adminUsersFile.getAbsolutePath() );
+ for (Stoppable worker : threadMap.keySet()) {
+ worker.setDone(true);
+ }
+ }
+
+ private void startAdminAuditors(BlockingQueue<Map<String, Object>> auditQueue, int workerCount) {
+ for (int x = 0; x < workerCount; x++) {
+ AuditWorker worker = new AuditWorker(auditQueue);
+ Thread workerThread = new Thread(worker, "AdminAuditor-" + x);
+ workerThread.start();
+ adminAuditThreads.put(worker, workerThread);
+ }
- jp.close();
}
- private String getType( Map<String, Object> entityProps ) {
- return ( String ) entityProps.get( PROPERTY_TYPE );
+ private void startAdminWorkers(BlockingQueue<Map<String, Object>> workQueue,
+ BlockingQueue<Map<String, Object>> auditQueue,
+ int workerCount) {
+
+ for (int x = 0; x < workerCount; x++) {
+ ImportAdminWorker worker = new ImportAdminWorker(workQueue, auditQueue);
+ Thread workerThread = new Thread(worker, "AdminWriter-" + x);
+ workerThread.start();
+ adminWriteThreads.put(worker, workerThread);
+ }
}
- private UUID getId( Map<String, Object> entityProps ) {
- return UUID.fromString( ( String ) entityProps.get( PROPERTY_UUID ) );
+ private String getType(Map<String, Object> entityProps) {
+ return (String) entityProps.get(PROPERTY_TYPE);
}
- private void validateStartArray( JsonToken token ) {
- if ( token != JsonToken.START_ARRAY ) {
- throw new RuntimeException( "Token should be START ARRAY but it is:" + token.asString() );
+ private UUID getId(Map<String, Object> entityProps) {
+ return UUID.fromString((String) entityProps.get(PROPERTY_UUID));
+ }
+
+
+ private void validateStartArray(JsonToken token) {
+ if (token != JsonToken.START_ARRAY) {
+ throw new RuntimeException("Token should be START ARRAY but it is:" + token.asString());
}
}
- private JsonParser getJsonParserForFile( File organizationFile ) throws Exception {
- JsonParser jp = jsonFactory.createJsonParser( organizationFile );
- jp.setCodec( new ObjectMapper() );
+ private JsonParser getJsonParserForFile(File organizationFile) throws Exception {
+ JsonParser jp = jsonFactory.createJsonParser(organizationFile);
+ jp.setCodec(new ObjectMapper());
return jp;
}
@@ -218,66 +293,82 @@ public class ImportAdmins extends ToolBase {
/**
* Import collections. Collections files are named: collections.<application_name>.Timestamp.json
*/
- private void importMetadata() throws Exception {
+ private void importMetadata(int writeThreadCount) throws Exception {
String[] fileNames = importDir.list(
- new PrefixFileFilter( ExportAdmins.ADMIN_USER_METADATA_PREFIX + "." ) );
- logger.info( "Metadata files to read: " + fileNames.length );
+ new PrefixFileFilter(ExportAdmins.ADMIN_USER_METADATA_PREFIX + "."));
+ logger.info("Metadata files to read: " + fileNames.length);
- for ( String fileName : fileNames ) {
+ for (String fileName : fileNames) {
try {
- importMetadata( fileName );
- }
- catch ( Exception e ) {
- logger.warn( "Unable to import metadata file: " + fileName, e );
+ importMetadata(fileName, writeThreadCount);
+ } catch (Exception e) {
+ logger.warn("Unable to import metadata file: " + fileName, e);
}
}
}
+ private void startMetadataWorkers(BlockingQueue<ImportMetadataTask> workQueue, int writeThreadCount) {
- @SuppressWarnings( "unchecked" )
- private void importMetadata( String fileName ) throws Exception {
+ for (int x = 0; x < writeThreadCount; x++) {
+ ImportMetadataWorker worker = new ImportMetadataWorker(workQueue);
+ Thread workerThread = new Thread(worker, "ImportMetadataTask-" + x);
+ workerThread.start();
+ metadataWorkerThreadMap.put(worker, workerThread);
+ }
+ }
- EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
- File metadataFile = new File( importDir, fileName );
+ @SuppressWarnings("unchecked")
+ private void importMetadata(String fileName, int writeThreads) throws Exception {
+
+ EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
- logger.info( "----- Loading metadata file: " + metadataFile.getAbsolutePath() );
+ File metadataFile = new File(importDir, fileName);
- JsonParser jp = getJsonParserForFile( metadataFile );
+ logger.info("----- Loading metadata file: " + metadataFile.getAbsolutePath());
+
+ JsonParser jp = getJsonParserForFile(metadataFile);
JsonToken jsonToken = null; // jp.nextToken();// START_OBJECT this is the outer hashmap
int depth = 1;
- while ( depth > 0 ) {
+ BlockingQueue<ImportMetadataTask> workQueue = new LinkedBlockingQueue<ImportMetadataTask>();
+ startMetadataWorkers(workQueue, writeThreads);
+
+ while (depth > 0) {
jsonToken = jp.nextToken();
- if ( jsonToken == null ) {
+ if (jsonToken == null) {
logger.info("token is null, breaking");
break;
}
- if (jsonToken.equals( JsonToken.START_OBJECT )) {
+ if (jsonToken.equals(JsonToken.START_OBJECT)) {
depth++;
- } else if (jsonToken.equals( JsonToken.END_OBJECT )) {
+ } else if (jsonToken.equals(JsonToken.END_OBJECT)) {
depth--;
}
- if (jsonToken.equals( JsonToken.FIELD_NAME ) && depth == 2 ) {
+ if (jsonToken.equals(JsonToken.FIELD_NAME) && depth == 2) {
jp.nextToken();
String entityOwnerId = jp.getCurrentName();
- EntityRef entityRef = em.getRef( UUID.fromString( entityOwnerId ) );
+ EntityRef entityRef = em.getRef(UUID.fromString(entityOwnerId));
- Map<String, Object> metadata = (Map<String, Object>)jp.readValueAs( Map.class );
- importEntityMetadata( em, entityRef, metadata );
+ Map<String, Object> metadata = (Map<String, Object>) jp.readValueAs(Map.class);
+
+ workQueue.put(new ImportMetadataTask(entityRef, metadata));
+// importEntityMetadata(em, entityRef, metadata);
}
}
- logger.info( "----- End of metadata -----" );
+ waitForQueueAndMeasure(workQueue, metadataWorkerThreadMap, "Metadata Load");
+
+ logger.info("----- End of metadata -----");
jp.close();
}
@@ -287,113 +378,113 @@ public class ImportAdmins extends ToolBase {
*/
@SuppressWarnings("unchecked")
private void importEntityMetadata(
- EntityManager em, EntityRef entityRef, Map<String, Object> metadata ) throws Exception {
+ EntityManager em, EntityRef entityRef, Map<String, Object> metadata) throws Exception {
- Map<String, Object> connectionsMap = (Map<String, Object>) metadata.get( "connections" );
+ Map<String, Object> connectionsMap = (Map<String, Object>) metadata.get("connections");
if (connectionsMap != null && !connectionsMap.isEmpty()) {
for (String type : connectionsMap.keySet()) {
try {
- UUID uuid = UUID.fromString( (String) connectionsMap.get( type ) );
- EntityRef connectedEntityRef = em.getRef( uuid );
- em.createConnection( entityRef, type, connectedEntityRef );
+ UUID uuid = UUID.fromString((String) connectionsMap.get(type));
+ EntityRef connectedEntityRef = em.getRef(uuid);
+ em.createConnection(entityRef, type, connectedEntityRef);
- logger.debug( "Creating connection from {} type {} target {}",
+ logger.debug("Creating connection from {} type {} target {}",
new Object[]{entityRef, type, connectedEntityRef});
} catch (Exception e) {
if (logger.isDebugEnabled()) {
- logger.error( "Error importing connection of type "
- + type + " for user " + entityRef.getUuid(), e );
+ logger.error("Error importing connection of type "
+ + type + " for user " + entityRef.getUuid(), e);
} else {
- logger.error( "Error importing connection of type "
- + type + " for user " + entityRef.getUuid() );
+ logger.error("Error importing connection of type "
+ + type + " for user " + entityRef.getUuid());
}
}
}
}
- Map<String, Object> dictionariesMap = (Map<String, Object>) metadata.get( "dictionaries" );
+ Map<String, Object> dictionariesMap = (Map<String, Object>) metadata.get("dictionaries");
if (dictionariesMap != null && !dictionariesMap.isEmpty()) {
for (String name : dictionariesMap.keySet()) {
try {
- Map<String, Object> dictionary = (Map<String, Object>) dictionariesMap.get( name );
- em.addMapToDictionary( entityRef, name, dictionary );
+ Map<String, Object> dictionary = (Map<String, Object>) dictionariesMap.get(name);
+ em.addMapToDictionary(entityRef, name, dictionary);
logger.debug("Creating dictionary for {} name {} map {}",
- new Object[] { entityRef, name, dictionary });
+ new Object[]{entityRef, name, dictionary});
} catch (Exception e) {
if (logger.isDebugEnabled()) {
- logger.error( "Error importing dictionary name "
- + name + " for user " + entityRef.getUuid(), e );
+ logger.error("Error importing dictionary name "
+ + name + " for user " + entityRef.getUuid(), e);
} else {
- logger.error( "Error importing dictionary name "
- + name + " for user " + entityRef.getUuid() );
+ logger.error("Error importing dictionary name "
+ + name + " for user " + entityRef.getUuid());
}
}
}
}
- List<String> collectionsList = (List<String>) metadata.get( "collections" );
+ List<String> collectionsList = (List<String>) metadata.get("collections");
if (collectionsList != null && !collectionsList.isEmpty()) {
for (String name : collectionsList) {
try {
- UUID uuid = UUID.fromString( (String) connectionsMap.get( name ) );
- EntityRef collectedEntityRef = em.getRef( uuid );
- em.addToCollection( entityRef, name, collectedEntityRef );
+ UUID uuid = UUID.fromString((String) connectionsMap.get(name));
+ EntityRef collectedEntityRef = em.getRef(uuid);
+ em.addToCollection(entityRef, name, collectedEntityRef);
logger.debug("Add to collection of {} name {} entity {}",
- new Object[] { entityRef, name, collectedEntityRef });
+ new Object[]{entityRef, name, collectedEntityRef});
} catch (Exception e) {
if (logger.isDebugEnabled()) {
- logger.error( "Error adding to collection "
- + name + " for user " + entityRef.getUuid(), e );
+ logger.error("Error adding to collection "
+ + name + " for user " + entityRef.getUuid(), e);
} else {
- logger.error( "Error adding to collection "
- + name + " for user " + entityRef.getUuid() );
+ logger.error("Error adding to collection "
+ + name + " for user " + entityRef.getUuid());
}
}
}
}
- List<Object> organizationsList = (List<Object>) metadata.get( "organizations" );
+ List<Object> organizationsList = (List<Object>) metadata.get("organizations");
if (organizationsList != null && !organizationsList.isEmpty()) {
- User user = em.get( entityRef, User.class );
+ User user = em.get(entityRef, User.class);
- if ( user == null ) {
+ if (user == null) {
logger.error("User with uuid={} not found, not adding to organizations");
} else {
- final UserInfo userInfo = managementService.getAdminUserByEmail( user.getEmail() );
+ final UserInfo userInfo = managementService.getAdminUserByEmail(user.getEmail());
for (Object orgObject : organizationsList) {
Map<String, Object> orgMap = (Map<String, Object>) orgObject;
- UUID orgUuid = UUID.fromString( (String) orgMap.get( "uuid" ) );
- String orgName = (String) orgMap.get( "name" );
+ UUID orgUuid = UUID.fromString((String) orgMap.get("uuid"));
+ String orgName = (String) orgMap.get("name");
// create org only if it does not exist
- OrganizationInfo orgInfo = managementService.getOrganizationByUuid( orgUuid );
+ OrganizationInfo orgInfo = managementService.getOrganizationByUuid(orgUuid);
if (orgInfo == null) {
try {
- managementService.createOrganization( orgUuid, orgName, userInfo, false );
- orgInfo = managementService.getOrganizationByUuid( orgUuid );
+ managementService.createOrganization(orgUuid, orgName, userInfo, false);
+ orgInfo = managementService.getOrganizationByUuid(orgUuid);
- logger.debug( "Created new org {} for user {}",
- new Object[]{orgInfo.getName(), user.getEmail()} );
+ logger.debug("Created new org {} for user {}",
+ new Object[]{orgInfo.getName(), user.getEmail()});
} catch (DuplicateUniquePropertyExistsException dpee) {
- logger.error( "Org {} already exists", orgName );
+ logger.error("Org {} already exists", orgName);
}
} else {
- managementService.addAdminUserToOrganization( userInfo, orgInfo, false );
- logger.debug( "Added user {} to org {}", new Object[]{user.getEmail(), orgName} );
+ managementService.addAdminUserToOrganization(userInfo, orgInfo, false);
+ logger.debug("Added user {} to org {}", new Object[]{user.getEmail(), orgName});
}
}
}
@@ -404,18 +495,229 @@ public class ImportAdmins extends ToolBase {
/**
* Open up the import directory based on <code>importDir</code>
*/
- private void openImportDirectory( CommandLine line ) {
+ private void openImportDirectory(CommandLine line) {
+
+ boolean hasInputDir = line.hasOption(INPUT_DIR);
+
+ if (hasInputDir) {
+ importDir = new File(line.getOptionValue(INPUT_DIR));
+ } else {
+ importDir = new File(DEFAULT_INPUT_DIR);
+ }
+
+ logger.info("Importing from:" + importDir.getAbsolutePath());
+ logger.info("Status. Exists: " + importDir.exists() + " - Readable: " + importDir.canRead());
+ }
- boolean hasInputDir = line.hasOption( INPUT_DIR );
- if ( hasInputDir ) {
- importDir = new File( line.getOptionValue( INPUT_DIR ) );
+ interface Stoppable {
+ void setDone(boolean done);
+ }
+
+ class AuditWorker implements Runnable, Stoppable {
+ private BlockingQueue<Map<String, Object>> workQueue;
+ private boolean done;
+
+ public AuditWorker(BlockingQueue<Map<String, Object>> workQueue) {
+ this.workQueue = workQueue;
}
- else {
- importDir = new File( DEFAULT_INPUT_DIR );
+
+ @Override
+ public void setDone(boolean done) {
+ this.done = done;
}
- logger.info( "Importing from:" + importDir.getAbsolutePath() );
- logger.info( "Status. Exists: " + importDir.exists() + " - Readable: " + importDir.canRead() );
+ @Override
+ public void run() {
+ int count = 0;
+
+ EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+
+ long durationSum = 0;
+
+ while (!done) {
+ try {
+ Map<String, Object> entityProps = this.workQueue.poll(30, TimeUnit.SECONDS);
+
+ if (entityProps == null) {
+ logger.warn("Reading from AUDIT queue was null!");
+ Thread.sleep(1000);
+ continue;
+ }
+
+ count++;
+ long startTime = System.currentTimeMillis();
+
+ UUID uuid = (UUID) entityProps.get(PROPERTY_UUID);
+ String type = getType(entityProps);
+
+ if (em.get(uuid) == null) {
+ logger.error("Holy hell, we wrote an entity and it's missing. " +
+ "Entity Id was {} and type is {}", uuid, type);
+ System.exit(1);
+ }
+
+ echo(entityProps);
+
+ long stopTime = System.currentTimeMillis();
+
+ long duration = stopTime - startTime;
+ durationSum += duration;
+ logger.debug(String.format("Audited [%s]th admin", count));
+ logger.info(String.format("Average Audit Rate: %s(ms)", durationSum / count));
+
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ logger.warn("Done!");
+ }
+ }
+
+
+ class ImportMetadataTask {
+ public EntityRef entityRef;
+ public Map<String, Object> metadata;
+
+ public ImportMetadataTask(EntityRef entityRef, Map<String, Object> metadata) {
+ this.entityRef = entityRef;
+ this.metadata = metadata;
+ }
+ }
+
+ class ImportMetadataWorker implements Runnable, Stoppable {
+ private BlockingQueue<ImportMetadataTask> workQueue;
+ private boolean done = false;
+
+ public ImportMetadataWorker(final BlockingQueue<ImportMetadataTask> workQueue) {
+ this.workQueue = workQueue;
+
+ }
+
+ @Override
+ public void setDone(boolean done) {
+ this.done = done;
+ }
+
+ @Override
+ public void run() {
+ int count = 0;
+
+ EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+
+ long durationSum = 0;
+
+ while (!done) {
+ try {
+ ImportMetadataTask task = this.workQueue.poll(30, TimeUnit.SECONDS);
+
+ if (task == null) {
+ logger.warn("Reading from metadata queue was null!");
+ Thread.sleep(1000);
+ continue;
+ }
+
+ count++;
+ long startTime = System.currentTimeMillis();
+ importEntityMetadata(em, task.entityRef, task.metadata);
+ long stopTime = System.currentTimeMillis();
+
+ long duration = stopTime - startTime;
+ durationSum += duration;
+ logger.debug(String.format("Imported [%s]th metadata", count));
+ logger.info(String.format("Average metadata Imported Rate: %s(ms)", durationSum / count));
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ logger.debug("EXCEPTION", e);
+ }
+ }
+
+ logger.warn("Done!");
+ }
+ }
+
+
+ class ImportAdminWorker implements Runnable, Stoppable {
+
+ private BlockingQueue<Map<String, Object>> workQueue;
+ private BlockingQueue<Map<String, Object>> auditQueue;
+ private boolean done = false;
+
+
+ public ImportAdminWorker(final BlockingQueue<Map<String, Object>> workQueue,
+ final BlockingQueue<Map<String, Object>> auditQueue) {
+ logger.info("New Worker!");
+ this.workQueue = workQueue;
+ this.auditQueue = auditQueue;
+ }
+
+ @Override
+ public void setDone(boolean done) {
+ this.done = done;
+ }
+
+ @Override
+ public void run() {
+ int count = 0;
+
+ EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+
+ long durationSum = 0;
+
+ while (!done) {
+
+ try {
+
+ Map<String, Object> entityProps = this.workQueue.poll(30, TimeUnit.SECONDS);
+
+ if (entityProps == null) {
+ logger.warn("Reading from admin import queue was null!");
+ Thread.sleep(1000);
+ continue;
+ }
+
+ // Import/create the entity
+ UUID uuid = getId(entityProps);
+ String type = getType(entityProps);
+
+
+ try {
+ long startTime = System.currentTimeMillis();
+ em.create(uuid, type, entityProps);
+ auditQueue.put(entityProps);
+ long stopTime = System.currentTimeMillis();
+
+ long duration = stopTime - startTime;
+ durationSum += duration;
+
+ count++;
+ logger.debug(String.format("Imported [%s]th admin user %s / %s", count, uuid, entityProps.get("username")));
+ logger.info(String.format("Average Creation Rate: %s(ms)", durationSum / count));
+
+ if (count % 100 == 0) {
+ logger.info("Imported {} admin users", count);
+ }
+ } catch (DuplicateUniquePropertyExistsException de) {
+ logger.warn("Unable to create entity. It appears to be a duplicate: " +
+ "id={}, type={}, name={}, username={}",
+ new Object[]{uuid, type, entityProps.get("name"), entityProps.get("username")});
+ if (logger.isDebugEnabled()) {
+ logger.debug("Exception", de);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ }
+
+ logger.warn("Done!");
+ }
}
}
[02/13] incubator-usergrid git commit: Use multiple threads to read
data from Cassandra and one thread to write that data to the two output
files.
Posted by sn...@apache.org.
Use multiple threads to read data from Cassandra and one thread to write that data to the two output files.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0631bc6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0631bc6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0631bc6b
Branch: refs/heads/two-dot-o
Commit: 0631bc6b9d0de0e9022284469ac32a0a2dce5d38
Parents: 5fbe31f
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Jun 11 17:08:52 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Jun 11 17:08:52 2015 -0400
----------------------------------------------------------------------
.../org/apache/usergrid/tools/ExportAdmins.java | 481 +++++++++++++------
1 file changed, 340 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0631bc6b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index 8831b89..90d3726 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@ -18,39 +18,61 @@
package org.apache.usergrid.tools;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
import com.google.common.collect.BiMap;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.Results.Level;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.utils.StringUtils;
import org.codehaus.jackson.JsonGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.commons.cli.CommandLine;
-
-import org.apache.usergrid.persistence.ConnectionRef;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.Results.Level;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
+import java.util.*;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
/**
+ * Export Admin Users and metadata including organizations.
+ *
* java -jar usergrid-tools.jar ExportAdmins
*/
public class ExportAdmins extends ExportingToolBase {
static final Logger logger = LoggerFactory.getLogger( ExportAdmins.class );
-
public static final String ADMIN_USERS_PREFIX = "admin-users";
public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";
+ private static final String READ_THREAD_COUNT = "readThreads";
+ private int readThreadCount;
+
+
+ /**
+ * Represents an AdminUser that has been read and is ready for export.
+ */
+ class AdminUserWriteTask {
+ Entity adminUser;
+ Map<String, List<UUID>> collectionsByName;
+ Map<String, List<ConnectionRef>> connectionsByType;
+ Map<String, Map<Object, Object>> dictionariesByName;
+ BiMap<UUID, String> orgNamesByUuid;
+ }
+
+ /**
+ * Export admin users using multiple threads.
+ * <p/>
+ * How it works:
+ * In main thread we query for IDs of all admin users, add each ID to read queue.
+ * Read-queue workers read admin user data, add data to write queue.
+ * One write-queue worker reads data writes to file.
+ */
@Override
- public void runTool( CommandLine line ) throws Exception {
+ public void runTool(CommandLine line) throws Exception {
startSpring();
setVerbose( line );
@@ -60,213 +82,390 @@ public class ExportAdmins extends ExportingToolBase {
outputDir = createOutputParentDir();
logger.info( "Export directory: " + outputDir.getAbsolutePath() );
- exportAdminUsers();
- }
+ if (StringUtils.isNotEmpty( line.getOptionValue( READ_THREAD_COUNT ) )) {
+ try {
+ readThreadCount = Integer.parseInt( line.getOptionValue( READ_THREAD_COUNT ) );
+ } catch (NumberFormatException nfe) {
+ logger.error( "-" + READ_THREAD_COUNT + " must be specified as an integer. Aborting..." );
+ return;
+ }
+ } else {
+ readThreadCount = 20;
+ }
+ // start write queue worker
- private void exportAdminUsers() throws Exception {
+ BlockingQueue<AdminUserWriteTask> writeQueue = new LinkedBlockingQueue<AdminUserWriteTask>();
+ AdminUserWriter adminUserWriter = new AdminUserWriter( writeQueue );
+ Thread writeThread = new Thread( adminUserWriter );
+ writeThread.start();
+ logger.debug( "Write thread started" );
- int count = 0;
+ // start read queue workers
+ BlockingQueue<UUID> readQueue = new LinkedBlockingQueue<UUID>();
+ List<AdminUserReader> readers = new ArrayList<AdminUserReader>();
+ for (int i = 0; i < readThreadCount; i++) {
+ AdminUserReader worker = new AdminUserReader( readQueue, writeQueue );
+ Thread readerThread = new Thread( worker, "AdminUserReader-" + i );
+ readerThread.start();
+ readers.add( worker );
+ }
+ logger.debug( readThreadCount + " read worker threads started" );
+
+ // query for IDs, add each to read queue
+
+ Query query = new Query();
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Level.IDS );
EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
+ Results ids = em.searchCollection( em.getApplicationRef(), "users", query );
- // write one JSON file for management application users
+ while (ids.size() > 0) {
+ for (UUID uuid : ids.getIds()) {
+ readQueue.add( uuid );
+ logger.debug( "Added uuid to readQueue: " + uuid );
+ }
+ if (ids.getCursor() == null) {
+ break;
+ }
+ query.setCursor( ids.getCursor() );
+ ids = em.searchCollection( em.getApplicationRef(), "users", query );
+ }
- JsonGenerator usersFile =
- getJsonGenerator( createOutputFile( ADMIN_USERS_PREFIX, em.getApplication().getName() ) );
- usersFile.writeStartArray();
+ adminUserWriter.setDone( true );
+ for (AdminUserReader aur : readers) {
+ aur.setDone( true );
+ }
- // write one JSON file for metadata: collections, connections and dictionaries of those users
+ logger.debug( "Waiting for write thread to complete" );
+ writeThread.join();
+ }
- JsonGenerator metadataFile =
- getJsonGenerator( createOutputFile( ADMIN_USER_METADATA_PREFIX, em.getApplication().getName() ) );
- metadataFile.writeStartObject();
- // query for and loop through all users in management application
+ @Override
+ @SuppressWarnings("static-access")
+ public Options createOptions() {
- Query query = new Query();
- query.setLimit( MAX_ENTITY_FETCH );
- query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+ Options options = super.createOptions();
- Results entities = em.searchCollection( em.getApplicationRef(), "users", query );
+ Option readThreads = OptionBuilder
+ .hasArg().withType(0).withDescription("Read Threads -" + READ_THREAD_COUNT).create(READ_THREAD_COUNT);
- while ( entities.size() > 0 ) {
+ options.addOption( readThreads );
+ return options;
+ }
- for ( Entity entity : entities ) {
- // write user to application file
- usersFile.writeObject( entity );
- echo( entity );
+ public class AdminUserReader implements Runnable {
- // write user's collections, connections, etc. to collections file
- saveEntityMetadata( metadataFile, em, null, entity );
+ private boolean done = true;
- logger.debug("Exported user {}", entity.getProperty( "email" ));
+ private final BlockingQueue<UUID> readQueue;
+ private final BlockingQueue<AdminUserWriteTask> writeQueue;
+
+ public AdminUserReader( BlockingQueue<UUID> readQueue, BlockingQueue<AdminUserWriteTask> writeQueue ) {
+ this.readQueue = readQueue;
+ this.writeQueue = writeQueue;
+ }
- count++;
- if ( count % 1000 == 0 ) {
- logger.info("Exported {} admin users", count);
- }
+ @Override
+ public void run() {
+ try {
+ readAndQueueAdminUsers();
+ } catch (Exception e) {
+ logger.error("Error reading data for export", e);
}
+ }
- if ( entities.getCursor() == null ) {
- break;
+
+ private void readAndQueueAdminUsers() throws Exception {
+
+ EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
+
+ while ( true ) {
+
+ UUID uuid = null;
+ try {
+ uuid = readQueue.poll( 30, TimeUnit.SECONDS );
+ logger.debug("Got item from entityId queue: " + uuid );
+
+ if ( uuid == null && done ) {
+ break;
+ }
+
+ Entity entity = em.get( uuid );
+
+ AdminUserWriteTask task = new AdminUserWriteTask();
+ task.adminUser = entity;
+
+ addCollectionsToTask( task, entity );
+ addDictionariesToTask( task, entity );
+ addConnectionsToTask( task, entity );
+ addOrganizationsToTask( task, entity );
+
+ writeQueue.add( task );
+
+ } catch ( Exception e ) {
+ logger.error("Error reading data for user " + uuid, e );
+ }
}
- query.setCursor( entities.getCursor() );
- entities = em.searchCollection( em.getApplicationRef(), "users", query );
}
- metadataFile.writeEndObject();
- metadataFile.close();
- usersFile.writeEndArray();
- usersFile.close();
+ private void addCollectionsToTask(AdminUserWriteTask task, Entity entity) throws Exception {
- logger.info("Exported total of {} admin users", count);
- }
+ EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
+ Set<String> collections = em.getCollections( entity );
+ if ((collections == null) || collections.isEmpty()) {
+ return;
+ }
+ task.collectionsByName = new HashMap<String, List<UUID>>();
- /**
- * Serialize and save the collection members of this <code>entity</code>
- *
- * @param em Entity Manager
- * @param application Application name
- * @param entity entity
- */
- private void saveEntityMetadata(
- JsonGenerator jg, EntityManager em, String application, Entity entity) throws Exception {
+ for (String collectionName : collections) {
+
+ List<UUID> uuids = task.collectionsByName.get( collectionName );
+ if ( uuids == null ) {
+ uuids = new ArrayList<UUID>();
+ task.collectionsByName.put( collectionName, uuids );
+ }
- saveCollections( jg, em, entity );
- saveConnections( entity, em, jg );
- saveOrganizations( entity, em, jg );
- saveDictionaries( entity, em, jg );
+ Results collectionMembers = em.getCollection( entity, collectionName, null, 100000, Level.IDS, false );
- // End the object if it was Started
- jg.writeEndObject();
- }
+ List<UUID> entityIds = collectionMembers.getIds();
+
+ if ((entityIds != null) && !entityIds.isEmpty()) {
+ for (UUID childEntityUUID : entityIds) {
+ uuids.add( childEntityUUID );
+ }
+ }
+ }
+ }
- private void saveCollections(JsonGenerator jg, EntityManager em, Entity entity) throws Exception {
+ private void addDictionariesToTask(AdminUserWriteTask task, Entity entity) throws Exception {
+ EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
- Set<String> collections = em.getCollections( entity );
+ Set<String> dictionaries = em.getDictionaries( entity );
- // Only create entry for Entities that have collections
- if ( ( collections == null ) || collections.isEmpty() ) {
- return;
+ task.dictionariesByName = new HashMap<String, Map<Object, Object>>();
+
+ for (String dictionary : dictionaries) {
+ Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
+ task.dictionariesByName.put( dictionary, dict );
+ }
}
- jg.writeFieldName( entity.getUuid().toString() );
- jg.writeStartObject();
- for ( String collectionName : collections ) {
+ private void addConnectionsToTask(AdminUserWriteTask task, Entity entity) throws Exception {
+ EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
- jg.writeFieldName( collectionName );
- // Start collection array.
- jg.writeStartArray();
+ task.connectionsByType = new HashMap<String, List<ConnectionRef>>();
- Results collectionMembers = em.getCollection( entity, collectionName, null, 100000, Level.IDS, false );
+ Set<String> connectionTypes = em.getConnectionTypes( entity );
+ for (String connectionType : connectionTypes) {
- List<UUID> entityIds = collectionMembers.getIds();
+ List<ConnectionRef> connRefs = task.connectionsByType.get( connectionType );
+ if ( connRefs == null ) {
+ connRefs = new ArrayList<ConnectionRef>();
+ }
- if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
- for ( UUID childEntityUUID : entityIds ) {
- jg.writeObject( childEntityUUID.toString() );
+ Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Level.IDS );
+ List<ConnectionRef> connections = results.getConnections();
+
+ for (ConnectionRef connectionRef : connections) {
+ connRefs.add( connectionRef );
}
}
+ }
- // End collection array.
- jg.writeEndArray();
+
+ private void addOrganizationsToTask(AdminUserWriteTask task, Entity entity) throws Exception {
+ task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( entity.getUuid() );
+ }
+
+ public void setDone(boolean done) {
+ this.done = done;
}
}
+ class AdminUserWriter implements Runnable {
- /**
- * Persists the connection for this entity.
- */
- private void saveDictionaries( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+ private boolean done = false;
- jg.writeFieldName( "dictionaries" );
- jg.writeStartObject();
+ private final BlockingQueue<AdminUserWriteTask> taskQueue;
- Set<String> dictionaries = em.getDictionaries( entity );
- for ( String dictionary : dictionaries ) {
+ public AdminUserWriter( BlockingQueue<AdminUserWriteTask> taskQueue ) {
+ this.taskQueue = taskQueue;
+ }
- Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
- // nothing to do
- if ( dict.isEmpty() ) {
- continue;
+ @Override
+ public void run() {
+ try {
+ writeEntities();
+ } catch (Exception e) {
+ logger.error("Error writing export data", e);
}
+ }
+
+
+ private void writeEntities() throws Exception {
+ EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
+
+ // write one JSON file for management application users
+ JsonGenerator usersFile =
+ getJsonGenerator( createOutputFile( ADMIN_USERS_PREFIX, em.getApplication().getName() ) );
+ usersFile.writeStartArray();
+
+ // write one JSON file for metadata: collections, connections and dictionaries of those users
+ JsonGenerator metadataFile =
+ getJsonGenerator( createOutputFile( ADMIN_USER_METADATA_PREFIX, em.getApplication().getName() ) );
+ metadataFile.writeStartObject();
+
+ int count = 0;
+
+ while ( true ) {
+
+ try {
+ AdminUserWriteTask task = taskQueue.poll( 30, TimeUnit.SECONDS );
+ if ( task == null && done ) {
+ break;
+ }
- jg.writeFieldName( dictionary );
+ // write user to application file
+ usersFile.writeObject( task.adminUser );
+ //usersFile.writeEndObject();
+ echo( task.adminUser );
+ // write metadata to metadata file
+ saveCollections( metadataFile, task );
+ saveConnections( metadataFile, task );
+ saveOrganizations( metadataFile, task );
+ saveDictionaries( metadataFile, task );
+
+ logger.debug("Exported user {}", task.adminUser.getProperty( "email" ));
+
+ count++;
+ if ( count % 1000 == 0 ) {
+ logger.info("Exported {} admin users", count);
+ }
+
+
+ } catch (InterruptedException e) {
+ throw new Exception("Interrupted", e);
+ }
+ }
+
+ metadataFile.writeEndObject();
+ metadataFile.close();
+
+ usersFile.writeEndArray();
+ usersFile.close();
+ }
+
+
+ private void saveCollections( JsonGenerator jg, AdminUserWriteTask task ) throws Exception {
+
+ jg.writeFieldName( task.adminUser.getUuid().toString() );
jg.writeStartObject();
- for ( Map.Entry<Object, Object> entry : dict.entrySet() ) {
- jg.writeFieldName( entry.getKey().toString() );
- jg.writeObject( entry.getValue() );
+ for (String collectionName : task.collectionsByName.keySet() ) {
+
+ jg.writeFieldName( collectionName );
+ jg.writeStartArray();
+
+ List<UUID> entityIds = task.collectionsByName.get( collectionName );
+
+ if ((entityIds != null) && !entityIds.isEmpty()) {
+ for (UUID childEntityUUID : entityIds) {
+ jg.writeObject( childEntityUUID.toString() );
+ }
+ }
+
+ jg.writeEndArray();
}
+ }
+
+ private void saveDictionaries( JsonGenerator jg, AdminUserWriteTask task ) throws Exception {
+
+ jg.writeFieldName( "dictionaries" );
+ jg.writeStartObject();
+
+ for (String dictionary : task.dictionariesByName.keySet() ) {
+
+ Map<Object, Object> dict = task.dictionariesByName.get( dictionary );
+
+ if (dict.isEmpty()) {
+ continue;
+ }
+
+ jg.writeFieldName( dictionary );
+
+ jg.writeStartObject();
+
+ for (Map.Entry<Object, Object> entry : dict.entrySet()) {
+ jg.writeFieldName( entry.getKey().toString() );
+ jg.writeObject( entry.getValue() );
+ }
+
+ jg.writeEndObject();
+ }
jg.writeEndObject();
}
- jg.writeEndObject();
- }
- /**
- * Persists the outgoing connections for this entity.
- */
- private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+ private void saveConnections( JsonGenerator jg, AdminUserWriteTask task ) throws Exception {
- jg.writeFieldName( "connections" );
- jg.writeStartObject();
+ jg.writeFieldName( "connections" );
+ jg.writeStartObject();
- Set<String> connectionTypes = em.getConnectionTypes( entity );
- for ( String connectionType : connectionTypes ) {
+ for (String connectionType : task.connectionsByType.keySet() ) {
- jg.writeFieldName( connectionType );
- jg.writeStartArray();
+ jg.writeFieldName( connectionType );
+ jg.writeStartArray();
- Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Level.IDS );
- List<ConnectionRef> connections = results.getConnections();
+ List<ConnectionRef> connections = task.connectionsByType.get( connectionType );
+ for (ConnectionRef connectionRef : connections) {
+ jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
+ }
- for ( ConnectionRef connectionRef : connections ) {
- jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
+ jg.writeEndArray();
}
-
- jg.writeEndArray();
+ jg.writeEndObject();
}
- jg.writeEndObject();
- }
- /**
- * Persists the incoming connections for this entity.
- */
- private void saveOrganizations(Entity entity, EntityManager em, JsonGenerator jg) throws Exception {
+ private void saveOrganizations( JsonGenerator jg, AdminUserWriteTask task ) throws Exception {
- final BiMap<UUID, String> orgs = managementService.getOrganizationsForAdminUser( entity.getUuid() );
+ final BiMap<UUID, String> orgs = task.orgNamesByUuid;
- jg.writeFieldName( "organizations" );
+ jg.writeFieldName( "organizations" );
+
+ jg.writeStartArray();
- jg.writeStartArray();
+ for (UUID uuid : orgs.keySet()) {
- for ( UUID uuid : orgs.keySet() ) {
+ jg.writeStartObject();
- jg.writeStartObject();
+ jg.writeFieldName( "uuid" );
+ jg.writeObject( uuid );
- jg.writeFieldName( "uuid" );
- jg.writeObject( uuid );
+ jg.writeFieldName( "name" );
+ jg.writeObject( orgs.get( uuid ) );
- jg.writeFieldName( "name" );
- jg.writeObject( orgs.get( uuid ) );
+ jg.writeEndObject();
+ }
- jg.writeEndObject();
+ jg.writeEndArray();
}
- jg.writeEndArray();
+ public void setDone(boolean done) {
+ this.done = done;
+ }
}
-
}
[13/13] incubator-usergrid git commit: Merge branch 'master' into
two-dot-o
Posted by sn...@apache.org.
Merge branch 'master' into two-dot-o
Conflicts:
stack/core/src/main/resources/usergrid-core-context.xml
stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
stack/services/src/main/java/org/apache/usergrid/security/shiro/Realm.java
stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/81082f35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/81082f35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/81082f35
Branch: refs/heads/two-dot-o
Commit: 81082f3588279fa5c0c3495a36f9aee6bc41d2b7
Parents: e1cdc73 107a465
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Jun 19 10:42:11 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Jun 19 10:42:11 2015 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-core-context.xml | 2 +-
.../applications/ApplicationResourceIT.java | 112 ++--
.../cassandra/ManagementServiceImpl.java | 3 +-
.../apache/usergrid/management/EmailFlowIT.java | 6 +-
.../org/apache/usergrid/tools/ExportAdmins.java | 469 +++++++++++----
.../org/apache/usergrid/tools/ImportAdmins.java | 598 ++++++++++++++-----
6 files changed, 872 insertions(+), 318 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/81082f35/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --cc stack/core/src/main/resources/usergrid-core-context.xml
index 4424896,7e69e19..53ae4e2
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@@ -54,9 -54,7 +54,9 @@@
<constructor-arg value="${cassandra.url}" />
<!-- set the pool size if it's available. If not go with 50 -->
<property name="maxActive" value="${cassandra.connections:20}"/>
+ <property name="cassandraThriftSocketTimeout" value="${cassandra.thriftSocketTimeout:0}" />
+ <property name="useSocketKeepalive" value="${cassandra.useSocketKeepalive:false}" />
- <!-- <property name="clockResolution" ref="microsecondsTimeResolution" /> -->
+ <!-- Fails on Java 8: <property name="clockResolution" ref="microsecondsTimeResolution" />-->
<property name="opTimer" ref="taggedOpTimer"/>
<property name="loadBalancingPolicy" ref="loadBalancingPolicy"/>
</bean>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/81082f35/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
index 104d436,469bf60..8629ccb
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
@@@ -16,29 -16,32 +16,29 @@@
*/
package org.apache.usergrid.rest.applications;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.ws.rs.core.MediaType;
-
-import org.codehaus.jackson.JsonNode;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.apache.usergrid.cassandra.Concurrent;
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.rest.AbstractRestIT;
-import org.apache.usergrid.rest.management.organizations.OrganizationsResource;
-
-import org.apache.shiro.codec.Base64;
-
-import com.sun.jersey.api.client.ClientResponse;
+import com.fasterxml.jackson.databind.JsonNode;
- import com.fasterxml.jackson.databind.ObjectMapper;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.representation.Form;
+import org.apache.shiro.codec.Base64;
-
- import org.apache.usergrid.persistence.index.utils.UUIDUtils;
++import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.rest.test.resource2point0.AbstractRestIT;
+import org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt.OrganizationResource;
+import org.apache.usergrid.rest.test.resource2point0.model.*;
+import org.apache.usergrid.utils.MapUtils;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
++import java.util.HashMap;
++import java.util.Map;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
import static org.apache.usergrid.utils.MapUtils.hashMap;
+import static org.junit.Assert.*;
/**
@@@ -78,83 -71,45 +78,121 @@@ public class ApplicationResourceIT exte
@Test
public void applicationWithAppCredentials() throws Exception {
- String orgName = "MiXedApplicationResourceTest";
- String appName = "mgmt-org-app-test";
-
- //create new org
- Map payload = hashMap( "email", "test-user-1@mockserver.com" ).map( "username", "ApplicationAppCredsTest" )
- .map( "name", "App Creds User" )
- .map( "password", "password" )
- .map( "organization",
- orgName );
-
- resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
-
- //create new app
- Map<String, String> data = new HashMap<String, String>();
- data.put( "name", appName );
-
- JsonNode appdata = resource().path( "/management/orgs/" + orgName + "/applications" )
- .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, data );
-
-
- ApplicationInfo appInfo = setup.getMgmtSvc().getApplicationInfo(orgName+"/"+appName );//"test-organization/test-app" );
-
- //don't know what kind of creds these are but they sure aren't app creds...
- String clientId = setup.getMgmtSvc().getClientIdForApplication( appInfo.getId() );
- String clientSecret = setup.getMgmtSvc().getClientSecretForApplication( appInfo.getId() );
-
- JsonNode applicationCredentials = resource().path( "/"+orgName.toLowerCase()+"/"+appName+"/credentials" ).queryParam( "client_id", clientId )
+ //retrieve the credentials
+ Credentials appCredentials = getAppCredentials();
+
+ //retrieve the app using only the org credentials
+ ApiResponse apiResponse = this.app().getResource(false)
+ .queryParam("grant_type", "client_credentials")
+ .queryParam("client_id", appCredentials.getClientId())
+ .queryParam("client_secret", appCredentials.getClientSecret())
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON_TYPE)
+ .get(ApiResponse.class);
+ //assert that a valid response is returned without error
+ assertNotNull(apiResponse);
+ assertNull(apiResponse.getError());
+ }
- .queryParam( "client_secret", clientSecret ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+ /**
+ * Retrieve an collection using the application client credentials
+ */
+ @Test
+ public void applicationCollectionWithAppCredentials() throws Exception {
+
+ //retrieve the credentials
+ Credentials appCredentials = getAppCredentials();
+
+ //retrieve the app using only the org credentials
+ ApiResponse apiResponse = this.app().collection( "roles" ).getResource( false )
+ .queryParam("grant_type", "client_credentials")
+ .queryParam("client_id", appCredentials.getClientId())
+ .queryParam("client_secret", appCredentials.getClientSecret())
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON_TYPE)
+ .get(ApiResponse.class);
+ //assert that a valid response is returned without error
+ assertNotNull(apiResponse);
+ assertNull(apiResponse.getError());
+
+ Collection roles = new Collection(apiResponse);
+ //assert that we have the correct number of default roles
+ assertEquals(3, roles.getNumOfEntities());
+ }
- /**
- * Retrieve an collection using the application client credentials
- */
- @Test
- public void applicationCollectionWithAppToken() throws Exception {
-
- Credentials appCredentials = getAppCredentials();
-
- QueryParameters queryParameters = new QueryParameters();
- queryParameters.addParam("grant_type", "client_credentials");
- queryParameters.addParam("client_id", appCredentials.getClientId());
- queryParameters.addParam("client_secret", appCredentials.getClientSecret());
-
- Entity testUser = new Entity( );
- testUser.chainPut( "name","temp" ).chainPut( "password","temp1" );
-
- Token appToken = this.app().token().post(new Token("client_credentials",appCredentials.getClientId(),appCredentials.getClientSecret() ));
-
-
- //retrieve the credentials
-
- //retrieve the app using only the org credentials
- ApiResponse apiResponse = this.app().collection( "roles" ).getResource( true, appToken )
- .accept( MediaType.APPLICATION_JSON )
- .type(MediaType.APPLICATION_JSON_TYPE)
- .get(ApiResponse.class);
- //assert that a valid response is returned without error
- assertNotNull(apiResponse);
- assertNull(apiResponse.getError());
-
- Collection roles = new Collection(apiResponse);
- //assert that we have the correct number of default roles
- assertEquals(3, roles.getNumOfEntities());
- }
- JsonNode node = resource().path( "/"+orgName.toLowerCase()+"/"+appName+"/users" ).queryParam( "client_id", applicationCredentials.get( "credentials" ).get( "client_id" ).asText())
- .queryParam( "client_secret", applicationCredentials.get( "credentials" ).get( "client_secret" ).asText() ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
++ // TODO: rewrite with new framework in two-dot-o-dev
++// /**
++// * Retrieve an collection using the application client credentials
++// */
++// @Test
++// public void applicationCollectionWithAppToken() throws Exception {
++//
++// Credentials appCredentials = getAppCredentials();
++//
++// QueryParameters queryParameters = new QueryParameters();
++// queryParameters.addParam("grant_type", "client_credentials");
++// queryParameters.addParam("client_id", appCredentials.getClientId());
++// queryParameters.addParam("client_secret", appCredentials.getClientSecret());
++//
++// Entity testUser = new Entity( );
++// testUser.chainPut( "name","temp" ).chainPut( "password","temp1" );
++//
++// Token appToken = this.app().token().post(
++// new Token("client_credentials",appCredentials.getClientId(),appCredentials.getClientSecret() ));
++//
++// String orgName = "MiXedApplicationResourceTest";
++// String appName = "mgmt-org-app-test";
++//
++// //create new org
++// Map payload = hashMap( "email", "test-user-1@mockserver.com" ).map( "username", "ApplicationAppCredsTest" )
++// .map( "name", "App Creds User" )
++// .map( "password", "password" )
++// .map( "organization", orgName );
++//
++// resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
++// .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
++//
++// //create new app
++// Map<String, String> data = new HashMap<String, String>();
++// data.put( "name", appName );
++//
++// JsonNode appdata = resource().path( "/management/orgs/" + orgName + "/applications" )
++// .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
++// .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, data );
++//
++//
++// ApplicationInfo appInfo = setup.getMgmtSvc().getApplicationInfo(orgName+"/"+appName );//"test-organization/test-app" );
++//
++// //don't know what kind of creds these are but they sure aren't app creds...
++// String clientId = setup.getMgmtSvc().getClientIdForApplication( appInfo.getId() );
++// String clientSecret = setup.getMgmtSvc().getClientSecretForApplication( appInfo.getId() );
++//
++// JsonNode applicationCredentials = resource().path( "/"+orgName.toLowerCase()+"/"+appName+"/credentials" ).queryParam( "client_id", clientId )
++//
++// .queryParam( "client_secret", clientSecret ).accept( MediaType.APPLICATION_JSON )
++// .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
++//
++// JsonNode node = resource().path( "/"+orgName.toLowerCase()+"/"+appName+"/users" ).queryParam( "client_id", applicationCredentials.get( "credentials" ).get( "client_id" ).asText())
++// .queryParam( "client_secret", applicationCredentials.get( "credentials" ).get( "client_secret" ).asText() ).accept( MediaType.APPLICATION_JSON )
++// .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
++//
++//
++// //retrieve the credentials
++//
++// //retrieve the app using only the org credentials
++// ApiResponse apiResponse = this.app().collection( "roles" ).getResource( true, appToken )
++// .accept( MediaType.APPLICATION_JSON )
++// .type(MediaType.APPLICATION_JSON_TYPE)
++// .get(ApiResponse.class);
++// //assert that a valid response is returned without error
++// assertNotNull(apiResponse);
++// assertNull(apiResponse.getError());
++//
++// Collection roles = new Collection(apiResponse);
++// //assert that we have the correct number of default roles
++// assertEquals(3, roles.getNumOfEntities());
++// }
- assertNotNull( node.get( "entities" ) );
- }
/**
* Verifies that we return JSON even when no accept header is specified.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/81082f35/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/81082f35/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/81082f35/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index 5bbad14,e781a1c..75064a2
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@@ -17,22 -17,24 +17,31 @@@
package org.apache.usergrid.tools;
- import org.apache.usergrid.persistence.index.query.Query;
-
import com.google.common.collect.BiMap;
import org.apache.commons.cli.CommandLine;
+ import org.apache.commons.cli.Option;
+ import org.apache.commons.cli.OptionBuilder;
+ import org.apache.commons.cli.Options;
-import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Results;
+ import org.apache.usergrid.persistence.Results.Level;
+ import org.apache.usergrid.persistence.cassandra.CassandraService;
++import org.apache.usergrid.persistence.index.query.Query;
+ import org.apache.usergrid.utils.StringUtils;
import org.codehaus.jackson.JsonGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+ import java.util.concurrent.BlockingQueue;
+ import java.util.concurrent.LinkedBlockingQueue;
+ import java.util.concurrent.TimeUnit;
/**
[07/13] incubator-usergrid git commit: Merge commit
'72fbb8bc7a85834e51b16b48608f45f7bd63c749'
Posted by sn...@apache.org.
Merge commit '72fbb8bc7a85834e51b16b48608f45f7bd63c749'
* commit '72fbb8bc7a85834e51b16b48608f45f7bd63c749':
Couple small fixes based on testing.
Use multiple threads to read data from Cassandra and one thread to write that data to the two output files.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3d8fc6fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3d8fc6fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3d8fc6fe
Branch: refs/heads/two-dot-o
Commit: 3d8fc6feccc29877bba65b14d64795b51929b3fa
Parents: a415af4 72fbb8b
Author: GERey <gr...@apigee.com>
Authored: Fri Jun 12 14:34:33 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Fri Jun 12 14:34:33 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/tools/ExportAdmins.java | 482 +++++++++++++------
1 file changed, 342 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
[11/13] incubator-usergrid git commit: Incrementing snapshot version
to 1.0.3-SNAPSHOT.
Posted by sn...@apache.org.
Incrementing snapshot version to 1.0.3-SNAPSHOT.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f0982b34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f0982b34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f0982b34
Branch: refs/heads/two-dot-o
Commit: f0982b347cda3c20a5d1097ab58592874886719f
Parents: 2d4589c
Author: Dave Johnson <dm...@apigee.com>
Authored: Sun Jun 14 08:08:33 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sun Jun 14 08:08:33 2015 -0400
----------------------------------------------------------------------
.usergridversion | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f0982b34/.usergridversion
----------------------------------------------------------------------
diff --git a/.usergridversion b/.usergridversion
index 276823b..99eda91 100644
--- a/.usergridversion
+++ b/.usergridversion
@@ -1 +1 @@
-1.0.2-SNAPSHOT
+1.0.3-SNAPSHOT
[03/13] incubator-usergrid git commit: Fix divide by zero error due
to rounding
Posted by sn...@apache.org.
Fix divide by zero error due to rounding
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a415af4a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a415af4a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a415af4a
Branch: refs/heads/two-dot-o
Commit: a415af4a5611c6e2303d2fc6917a1311c6e58719
Parents: c0b89db
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Jun 12 10:42:55 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Jun 12 10:42:55 2015 -0400
----------------------------------------------------------------------
.../src/main/java/org/apache/usergrid/tools/ImportAdmins.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a415af4a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
index 7d49459..d05c9a8 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
@@ -229,7 +229,7 @@ public class ImportAdmins extends ToolBase {
double rateLast = (double) processed / (timeDelta / 1000);
rateAverageSum += rateLast;
- long timeRemaining = sizeLast / (long) (rateAverageSum / iterationCounter);
+ long timeRemaining = (long) ( sizeLast / (rateAverageSum / iterationCounter) );
logger.info(
String.format("++PROGRESS (%s): sizeLast=%s nowSize=%s processed=%s rateLast=%s/s rateAvg=%s/s timeRemaining=%s(s)",
[12/13] incubator-usergrid git commit: Merge branch
'USERGRID-670-one-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid
Posted by sn...@apache.org.
Merge branch 'USERGRID-670-one-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/107a465e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/107a465e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/107a465e
Branch: refs/heads/two-dot-o
Commit: 107a465e5d6d609705bf3736c9b9a9507955386c
Parents: f0982b3 136381a
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Jun 16 09:48:33 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Jun 16 09:48:33 2015 -0600
----------------------------------------------------------------------
.../applications/ApplicationResourceIT.java | 37 ++++++++++++++++++--
.../cassandra/ManagementServiceImpl.java | 3 +-
.../apache/usergrid/security/shiro/Realm.java | 2 +-
.../apache/usergrid/management/EmailFlowIT.java | 6 ++--
4 files changed, 40 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[10/13] incubator-usergrid git commit: Updating CHANGELOG for 1.0.2
release.
Posted by sn...@apache.org.
Updating CHANGELOG for 1.0.2 release.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2d4589c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2d4589c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2d4589c8
Branch: refs/heads/two-dot-o
Commit: 2d4589c8b07fe1928c2e3cfe1acbf4cc2f718594
Parents: 10ca622
Author: Dave Johnson <dm...@apigee.com>
Authored: Sun Jun 14 08:08:33 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sun Jun 14 08:08:33 2015 -0400
----------------------------------------------------------------------
CHANGELOG | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2d4589c8/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 70ea277..11f113a 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,6 +1,9 @@
Usergrid 1.0.2
--------------------------------------------------------------------------------
+Usergrid 1.0.2
+--------------------------------------------------------------------------------
+
Usergrid 1.0.2
--------------------------------------------------------------------------------
[06/13] incubator-usergrid git commit: Added fixes to email flow It
due to having lower cased application names now.
Posted by sn...@apache.org.
Added fixes to email flow It due to having lower cased application names now.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/136381ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/136381ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/136381ad
Branch: refs/heads/two-dot-o
Commit: 136381ada93e14d4b8b171cc1f8dedc074e1b6cb
Parents: db83ff4
Author: GERey <gr...@apigee.com>
Authored: Fri Jun 12 14:33:56 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Fri Jun 12 14:33:56 2015 -0700
----------------------------------------------------------------------
.../test/java/org/apache/usergrid/management/EmailFlowIT.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/136381ad/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java b/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java
index ab4b34c..76847aa 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/EmailFlowIT.java
@@ -265,7 +265,7 @@ public class EmailFlowIT {
// activation url ok
String mailContent = ( String ) ( ( MimeMultipart ) activation.getContent() ).getBodyPart( 1 ).getContent();
LOG.info( mailContent );
- assertTrue( StringUtils.contains( mailContent, activation_url ) );
+ assertTrue( StringUtils.contains( mailContent.toLowerCase(), activation_url.toLowerCase() ) );
// token ok
String token = getTokenFromMessage( activation );
@@ -293,7 +293,7 @@ public class EmailFlowIT {
// resetpwd url ok
mailContent = ( String ) ( ( MimeMultipart ) reset.getContent() ).getBodyPart( 1 ).getContent();
LOG.info( mailContent );
- assertTrue( StringUtils.contains( mailContent, reset_url ) );
+ assertTrue( StringUtils.contains( mailContent.toLowerCase(), reset_url.toLowerCase() ) );
// token ok
token = getTokenFromMessage( reset );
@@ -344,7 +344,7 @@ public class EmailFlowIT {
// confirmation url ok
String mailContent = ( String ) ( ( MimeMultipart ) confirmation.getContent() ).getBodyPart( 1 ).getContent();
LOG.info( mailContent );
- assertTrue( StringUtils.contains( mailContent, confirmation_url ) );
+ assertTrue( StringUtils.contains( mailContent.toLowerCase(), confirmation_url.toLowerCase() ) );
// token ok
String token = getTokenFromMessage( confirmation );
[04/13] incubator-usergrid git commit: Added test that proves the
issue that we couldn't use application credentials. Fixed not being able to
get mixed case applications using application credentials in the realm. Fixed
the creation of mixed case applica
Posted by sn...@apache.org.
Added test that proves the issue that we couldn't use application credentials.
Fixed not being able to get mixed case applications using application credentials in the realm.
Fixed the creation of mixed case applications, now its all handled as lower case in the backend...for applications at least.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/db83ff47
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/db83ff47
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/db83ff47
Branch: refs/heads/two-dot-o
Commit: db83ff47c118c7c959ee425f8d2b5ce2e09c8c2c
Parents: a415af4
Author: GERey <gr...@apigee.com>
Authored: Fri Jun 12 12:29:50 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Fri Jun 12 12:29:50 2015 -0700
----------------------------------------------------------------------
.../applications/ApplicationResourceIT.java | 37 ++++++++++++++++++--
.../cassandra/ManagementServiceImpl.java | 3 +-
.../apache/usergrid/security/shiro/Realm.java | 2 +-
3 files changed, 37 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db83ff47/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
index e7af2f5..469bf60 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/ApplicationResourceIT.java
@@ -17,6 +17,7 @@
package org.apache.usergrid.rest.applications;
+import java.util.HashMap;
import java.util.Map;
import javax.ws.rs.core.MediaType;
@@ -28,6 +29,7 @@ import org.apache.usergrid.cassandra.Concurrent;
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.rest.AbstractRestIT;
+import org.apache.usergrid.rest.management.organizations.OrganizationsResource;
import org.apache.shiro.codec.Base64;
@@ -65,16 +67,45 @@ public class ApplicationResourceIT extends AbstractRestIT {
assertNotNull( node.get( "entities" ) );
}
+ //TODO: write a test so that its mixed case.
@Test
public void applicationWithAppCredentials() throws Exception {
- ApplicationInfo appInfo = setup.getMgmtSvc().getApplicationInfo( "test-organization/test-app" );
+ String orgName = "MiXedApplicationResourceTest";
+ String appName = "mgmt-org-app-test";
+
+ //create new org
+ Map payload = hashMap( "email", "test-user-1@mockserver.com" ).map( "username", "ApplicationAppCredsTest" )
+ .map( "name", "App Creds User" )
+ .map( "password", "password" )
+ .map( "organization",
+ orgName );
+
+ resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+
+ //create new app
+ Map<String, String> data = new HashMap<String, String>();
+ data.put( "name", appName );
+
+ JsonNode appdata = resource().path( "/management/orgs/" + orgName + "/applications" )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, data );
+
+ ApplicationInfo appInfo = setup.getMgmtSvc().getApplicationInfo(orgName+"/"+appName );//"test-organization/test-app" );
+
+ //don't know what kind of creds these are but they sure aren't app creds...
String clientId = setup.getMgmtSvc().getClientIdForApplication( appInfo.getId() );
String clientSecret = setup.getMgmtSvc().getClientSecretForApplication( appInfo.getId() );
- JsonNode node = resource().path( "/test-organization/test-app/users" ).queryParam( "client_id", clientId )
- .queryParam( "client_secret", clientSecret ).accept( MediaType.APPLICATION_JSON )
+ JsonNode applicationCredentials = resource().path( "/"+orgName.toLowerCase()+"/"+appName+"/credentials" ).queryParam( "client_id", clientId )
+
+ .queryParam( "client_secret", clientSecret ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+
+ JsonNode node = resource().path( "/"+orgName.toLowerCase()+"/"+appName+"/users" ).queryParam( "client_id", applicationCredentials.get( "credentials" ).get( "client_id" ).asText())
+ .queryParam( "client_secret", applicationCredentials.get( "credentials" ).get( "client_secret" ).asText() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
assertNotNull( node.get( "entities" ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db83ff47/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index b3204a7..b117be5 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -680,7 +680,8 @@ public class ManagementServiceImpl implements ManagementService {
* (and that organization is needed) if so.
*/
private String buildAppName( String applicationName, OrganizationInfo organization ) {
- return applicationName.contains( "/" ) ? applicationName : organization.getName() + "/" + applicationName;
+ return org.apache.commons.lang.StringUtils.lowerCase(
+ applicationName.contains( "/" ) ? applicationName : organization.getName() + "/" + applicationName);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db83ff47/stack/services/src/main/java/org/apache/usergrid/security/shiro/Realm.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/Realm.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/Realm.java
index 0f1f9e8..0188922 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/Realm.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/Realm.java
@@ -278,7 +278,7 @@ public class Realm extends AuthorizingRealm {
application = ( ( ApplicationPrincipal ) principal ).getApplication();
grant( info, principal, "applications:admin,access,get,put,post,delete:" + application.getId() );
- applicationSet.put( application.getId(), application.getName() );
+ applicationSet.put( application.getId(), application.getName().toLowerCase());
}
else if ( principal instanceof AdminUserPrincipal ) {
// AdminUserPrincipals are through basic auth and sessions
[09/13] incubator-usergrid git commit: Correct version for creating
1.0.2 RC2
Posted by sn...@apache.org.
Correct version for creating 1.0.2 RC2
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/10ca622c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/10ca622c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/10ca622c
Branch: refs/heads/two-dot-o
Commit: 10ca622cd19b38b7f289e58d82287af67230acf6
Parents: 00114cb
Author: Dave Johnson <dm...@apigee.com>
Authored: Sun Jun 14 08:00:25 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sun Jun 14 08:00:25 2015 -0400
----------------------------------------------------------------------
.usergridversion | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/10ca622c/.usergridversion
----------------------------------------------------------------------
diff --git a/.usergridversion b/.usergridversion
index 99eda91..276823b 100644
--- a/.usergridversion
+++ b/.usergridversion
@@ -1 +1 @@
-1.0.3-SNAPSHOT
+1.0.2-SNAPSHOT
[08/13] incubator-usergrid git commit: Comment out apparently
obsolete property that causes build to fail on Java 8.
Posted by sn...@apache.org.
Comment out apparently obsolete property that causes build to fail on Java 8.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/00114cb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/00114cb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/00114cb7
Branch: refs/heads/two-dot-o
Commit: 00114cb7009be14dceb6a5fbe4bb6bcae449a024
Parents: 3d8fc6f
Author: Dave Johnson <dm...@apigee.com>
Authored: Sun Jun 14 07:56:49 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sun Jun 14 07:56:49 2015 -0400
----------------------------------------------------------------------
stack/core/src/main/resources/usergrid-core-context.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/00114cb7/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index 515d642..7e69e19 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -54,7 +54,7 @@
<constructor-arg value="${cassandra.url}" />
<!-- set the pool size if it's available. If not go with 50 -->
<property name="maxActive" value="${cassandra.connections:20}"/>
- <property name="clockResolution" ref="microsecondsTimeResolution" />
+ <!-- Fails on Java 8: <property name="clockResolution" ref="microsecondsTimeResolution" />-->
<property name="opTimer" ref="taggedOpTimer"/>
<property name="loadBalancingPolicy" ref="loadBalancingPolicy"/>
</bean>
[05/13] incubator-usergrid git commit: Couple small fixes based on
testing.
Posted by sn...@apache.org.
Couple small fixes based on testing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/72fbb8bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/72fbb8bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/72fbb8bc
Branch: refs/heads/two-dot-o
Commit: 72fbb8bc7a85834e51b16b48608f45f7bd63c749
Parents: 0631bc6
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Jun 12 15:54:22 2015 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Jun 12 15:54:22 2015 -0400
----------------------------------------------------------------------
.../src/main/java/org/apache/usergrid/tools/ExportAdmins.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/72fbb8bc/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index 90d3726..e781a1c 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@ -337,7 +337,6 @@ public class ExportAdmins extends ExportingToolBase {
// write user to application file
usersFile.writeObject( task.adminUser );
- //usersFile.writeEndObject();
echo( task.adminUser );
// write metadata to metadata file
@@ -364,6 +363,8 @@ public class ExportAdmins extends ExportingToolBase {
usersFile.writeEndArray();
usersFile.close();
+
+ logger.info("Exported TOTAL {} admin users", count);
}
@@ -387,6 +388,8 @@ public class ExportAdmins extends ExportingToolBase {
jg.writeEndArray();
}
+
+ jg.writeEndObject();
}