You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/07/22 17:17:35 UTC
[1/3] incubator-usergrid git commit: Merge branch 'master' into
two-dot-o
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-872 [created] b5144d058
Merge branch 'master' into two-dot-o
Conflicts:
stack/core/pom.xml
stack/pom.xml
stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
stack/tools/src/main/resources/log4j.properties
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/066d7db4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/066d7db4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/066d7db4
Branch: refs/heads/USERGRID-872
Commit: 066d7db46786cc042cdc4fa28b56eb2f729561d1
Parents: 764a7c8 9b9f55a
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jul 22 09:44:58 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jul 22 09:44:58 2015 -0400
----------------------------------------------------------------------
stack/core/pom.xml | 1 -
stack/pom.xml | 2 +-
stack/tools/pom.xml | 6 +
.../org/apache/usergrid/tools/ExportAdmins.java | 117 ++--
.../org/apache/usergrid/tools/ExportApp.java | 536 +++++++++++++++++++
.../usergrid/tools/ExportDataCreator.java | 244 +++++++--
.../usergrid/tools/ExportingToolBase.java | 2 +-
.../org/apache/usergrid/tools/ImportAdmins.java | 226 ++++++--
.../org/apache/usergrid/tools/ToolBase.java | 2 +-
stack/tools/src/main/resources/log4j.properties | 5 +
.../apache/usergrid/tools/ExportAppTest.java | 118 ++++
.../usergrid/tools/ExportImportAdminsTest.java | 71 ++-
...adata.usergrid-management.1433331614293.json | 52 ++
...users.usergrid-management.1433331614293.json | 12 +
14 files changed, 1225 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/core/pom.xml
----------------------------------------------------------------------
diff --cc stack/core/pom.xml
index d91208d,f60dbc2..5e65ac3
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@@ -473,23 -573,16 +473,22 @@@
</dependency>
<dependency>
- <groupId>io.reactivex</groupId>
- <artifactId>rxjava</artifactId>
+ <groupId>com.netflix.rxjava</groupId>
+ <artifactId>rxjava-core</artifactId>
<version>${rx.version}</version>
</dependency>
-
-
<dependency>
- <groupId>io.reactivex</groupId>
+ <groupId>com.netflix.rxjava</groupId>
<artifactId>rxjava-math</artifactId>
- <version>1.0.0</version>
+ <version>${rx.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>com.clearspring.analytics</groupId>
+ <artifactId>stream</artifactId>
+ <version>2.7.0</version>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/pom.xml
----------------------------------------------------------------------
diff --cc stack/pom.xml
index cc61a7c,0e1b32d..cc39e04
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@@ -32,107 -32,84 +32,107 @@@
<description>Parent module for the Apache Usergrid Project</description>
<packaging>pom</packaging>
- <properties>
- <!-- =================================================================== -->
- <!-- Properties: Deployment Setting Defaults -->
- <!-- =================================================================== -->
- <!-- NOTE: override from the CLI or settings.xml -->
- <!-- NOTE: add server credentials config via settings -->
- <!-- NOTE: <settings> -->
- <!-- NOTE: <servers> -->
- <!-- NOTE: <server> -->
- <!-- NOTE: <id>usergrid.releases</id> -->
- <!-- NOTE: <username>akarasulu</username> -->
- <!-- NOTE: <password>*********</password> -->
- <!-- NOTE: </server> -->
- <!-- NOTE: <server> -->
- <!-- NOTE: <id>usergrid.snapshots</id> -->
- <!-- NOTE: <username>akarasulu</username> -->
- <!-- NOTE: <password>*********</password> -->
- <!-- NOTE: </server> -->
- <!-- NOTE: </servers> -->
- <!-- NOTE: -->
- <!-- NOTE: <profiles> -->
- <!-- NOTE: <profile> -->
- <!-- NOTE: <id>deployment</id> -->
- <!-- NOTE: <properties> -->
- <!-- NOTE: <release.repository.url> -->
- <!-- NOTE: https://to/your/custom/releases/repository -->
- <!-- NOTE: </release.repository.url> -->
- <!-- NOTE: <snapshot.repository.url> -->
- <!-- NOTE: https://to/your/custom/snapshots/repository -->
- <!-- NOTE: </shapshot.repository.url> -->
- <!-- NOTE: </properties> -->
- <!-- NOTE: </profile> -->
- <!-- NOTE: </profiles> -->
- <!-- NOTE: -->
- <!-- NOTE: <activeProfiles> -->
- <!-- NOTE: <activeProfile>deployment</activeProfile> -->
- <!-- NOTE: </activeProfiles> -->
- <!-- NOTE: </settings> -->
-
- <!-- =================================================================== -->
- <!-- Properties: General Settings -->
- <!-- =================================================================== -->
-
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-
- <!-- you can override these via MAVEN_OPTS -->
- <ug.heapmax>2048m</ug.heapmax>
- <ug.heapmin>2048m</ug.heapmin>
- <ug.argline>-Djava.awt.headless=true</ug.argline>
-
- <usergrid-custom-spring-properties>classpath:/usergrid-custom.properties</usergrid-custom-spring-properties>
- <usergrid-custom-spring-test-properties>classpath:/usergrid-custom-test.properties</usergrid-custom-spring-test-properties>
-
- <!-- =================================================================== -->
- <!-- Properties: Dependency Settings -->
- <!-- =================================================================== -->
-
- <amber-version>0.22-incubating</amber-version>
- <cassandra-version>1.2.12</cassandra-version>
- <hector-om-version>3.0-03</hector-om-version>
- <hector-version>1.1-4</hector-version>
- <hector-test-version>1.1-4</hector-test-version>
- <jackson-version>1.9.9</jackson-version>
- <jclouds.version>1.7.1</jclouds.version>
- <jersey-version>1.18</jersey-version>
- <junit-version>4.11</junit-version>
- <log4j-version>1.2.16</log4j-version>
- <metrics-version>2.1.2</metrics-version>
- <org.springframework.version>3.1.2.RELEASE</org.springframework.version>
- <shiro-version>1.2.0</shiro-version>
- <slf4j-version>1.6.1</slf4j-version>
- <snakeyaml-version>1.9</snakeyaml-version>
- <tomcat-version>7.0.42</tomcat-version>
- <antlr.version>3.4</antlr.version>
- <tika.version>1.4</tika.version>
- <metrics.version>3.0.0</metrics.version>
- <rx.version>1.0.12</rx.version>
- </properties>
+ <properties>
+ <!-- =================================================================== -->
+ <!-- Properties: Deployment Setting Defaults -->
+ <!-- =================================================================== -->
+ <!-- NOTE: override from the CLI or settings.xml -->
+ <!-- NOTE: add server credentials config via settings -->
+ <!-- NOTE: <settings> -->
+ <!-- NOTE: <servers> -->
+ <!-- NOTE: <server> -->
+ <!-- NOTE: <id>usergrid.releases</id> -->
+ <!-- NOTE: <username>akarasulu</username> -->
+ <!-- NOTE: <password>*********</password> -->
+ <!-- NOTE: </server> -->
+ <!-- NOTE: <server> -->
+ <!-- NOTE: <id>usergrid.snapshots</id> -->
+ <!-- NOTE: <username>akarasulu</username> -->
+ <!-- NOTE: <password>*********</password> -->
+ <!-- NOTE: </server> -->
+ <!-- NOTE: </servers> -->
+ <!-- NOTE: -->
+ <!-- NOTE: <profiles> -->
+ <!-- NOTE: <profile> -->
+ <!-- NOTE: <id>deployment</id> -->
+ <!-- NOTE: <properties> -->
+ <!-- NOTE: <release.repository.url> -->
+ <!-- NOTE: https://to/your/custom/releases/repository -->
+ <!-- NOTE: </release.repository.url> -->
+ <!-- NOTE: <snapshot.repository.url> -->
+ <!-- NOTE: https://to/your/custom/snapshots/repository -->
+ <!-- NOTE: </shapshot.repository.url> -->
+ <!-- NOTE: </properties> -->
+ <!-- NOTE: </profile> -->
+ <!-- NOTE: </profiles> -->
+ <!-- NOTE: -->
+ <!-- NOTE: <activeProfiles> -->
+ <!-- NOTE: <activeProfile>deployment</activeProfile> -->
+ <!-- NOTE: </activeProfiles> -->
+ <!-- NOTE: </settings> -->
+
+ <snapshot.repository.url>
+ https://repository.apache.org/content/repositories/snapshots
+ </snapshot.repository.url>
+ <release.repository.url>
+ https://repository.apache.org/service/local/staging/deploy/maven2
+ </release.repository.url>
+
+ <!-- =================================================================== -->
+ <!-- Properties: General Settings -->
+ <!-- =================================================================== -->
+
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+
+ <!-- you can override these via MAVEN_OPTS -->
+ <ug.heapmax>4096m</ug.heapmax>
+ <ug.heapmin>2048m</ug.heapmin>
+ <ug.argline>-Djava.awt.headless=true</ug.argline>
+
+ <usergrid-custom-spring-properties>classpath:/usergrid-deployment.properties</usergrid-custom-spring-properties>
+ <usergrid-custom-spring-test-properties>classpath:/usergrid-custom-test.properties</usergrid-custom-spring-test-properties>
+
+ <!-- =================================================================== -->
+ <!-- Properties: Dependency Settings -->
+ <!-- =================================================================== -->
+
+ <amber-version>0.22-incubating</amber-version>
+ <cassandra-version>1.2.18</cassandra-version>
+ <guava.version>18.0</guava.version>
+ <guice.version>4.0-beta5</guice.version>
+ <hector-om-version>3.0-03</hector-om-version>
+ <hector-version>1.1-4</hector-version>
+ <hector-test-version>1.1-4</hector-test-version>
+ <jackson-version>1.9.9</jackson-version>
+ <jackson-2-version>2.3.3</jackson-2-version>
+ <jclouds.version>1.8.0</jclouds.version>
+ <jersey-version>1.18.1</jersey-version>
+ <junit-version>4.12</junit-version>
+ <log4j-version>1.2.16</log4j-version>
+ <org.springframework.version>3.1.2.RELEASE</org.springframework.version>
+ <shiro-version>1.2.3</shiro-version>
+ <slf4j-version>1.6.1</slf4j-version>
- <snakeyaml-version>1.8</snakeyaml-version>
++ <snakeyaml-version>1.9</snakeyaml-version>
+ <tomcat-version>7.0.59</tomcat-version>
+ <antlr.version>3.4</antlr.version>
+ <tika.version>1.4</tika.version>
+ <mockito.version>1.10.8</mockito.version>
+
+ <!-- only use half the cores on the machine for testing -->
+ <usergrid.it.parallel>methods</usergrid.it.parallel>
+ <usergrid.it.reuseForks>true</usergrid.it.reuseForks>
+ <usergrid.it.forkCount>1</usergrid.it.forkCount>
+ <usergrid.it.threads>8</usergrid.it.threads>
+
+ <metrics.version>3.0.0</metrics.version>
+ <rx.version>0.19.6</rx.version>
+ <surefire.plugin.artifactName>surefire-junit47</surefire.plugin.artifactName>
+ <surefire.plugin.version>2.18.1</surefire.plugin.version>
+ <powermock.version>1.6.1</powermock.version>
+
+ <maven.build.timestamp.format>yyyy-MM-dd'T'HH-mm-ss'Z'</maven.build.timestamp.format>
+
+ </properties>
<licenses>
<license>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/tools/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index e175d01,d5dd42c..0bb74ab
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@@ -22,12 -23,13 +22,10 @@@ import org.apache.commons.cli.CommandLi
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
- import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
- import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.Results.Level;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.utils.StringUtils;
import org.codehaus.jackson.JsonGenerator;
import org.slf4j.Logger;
@@@ -57,17 -61,29 +55,29 @@@ import java.util.concurrent.atomic.Atom
* cassandra.lock.keyspace=My_Usergrid_Locks
*/
public class ExportAdmins extends ExportingToolBase {
-
+
static final Logger logger = LoggerFactory.getLogger( ExportAdmins.class );
-
++
public static final String ADMIN_USERS_PREFIX = "admin-users";
public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";
-
++
+ // map admin user UUID to list of organizations to which user belongs
+ private Map<UUID, List<Org>> userToOrgsMap = new HashMap<UUID, List<Org>>(50000);
+
+ private Map<String, UUID> orgNameToUUID = new HashMap<String, UUID>(50000);
-
++
+ private Set<UUID> orgsWritten = new HashSet<UUID>(50000);
-
++
+ private Set<UUID> duplicateOrgs = new HashSet<UUID>();
-
++
private static final String READ_THREAD_COUNT = "readThreads";
- private Map<String, List<Org>> orgMap = new HashMap<String, List<Org>>(80000);
private int readThreadCount;
- AtomicInteger count = new AtomicInteger( 0 );
+ AtomicInteger userCount = new AtomicInteger( 0 );
-
++
+ boolean ignoreInvalidUsers = false; // true to ignore users with no credentials or orgs
-
-
+
+
/**
* Represents an AdminUser that has been read and is ready for export.
*/
@@@ -169,10 -185,10 +179,10 @@@
while ( !done ) {
writeThread.join( 10000, 0 );
done = !writeThread.isAlive();
- logger.info( "Wrote {} users", count.get() );
+ logger.info( "Wrote {} users", userCount.get() );
}
}
-
+
@Override
@SuppressWarnings("static-access")
@@@ -207,10 -223,11 +217,11 @@@
organizations = em.searchCollection( em.getApplicationRef(), "groups", query );
for ( Entity organization : organizations.getEntities() ) {
execService.submit( new OrgMapWorker( organization ) );
+ count++;
}
- count++;
-
++
if ( count % 1000 == 0 ) {
- logger.info("Processed {} orgs for org map", count);
+ logger.info("Queued {} org map workers", count);
}
query.setCursor( organizations.getCursor() );
}
@@@ -218,8 -235,10 +229,10 @@@
execService.shutdown();
while ( !execService.awaitTermination( 10, TimeUnit.SECONDS ) ) {
- logger.info("Processed {} orgs for map", orgMap.size() );
+ logger.info( "Processed {} orgs for map", userToOrgsMap.size() );
}
-
++
+ logger.info("Org map complete, counted {} organizations", count);
}
@@@ -235,6 -254,7 +248,7 @@@
try {
final String orgName = orgEntity.getProperty( "path" ).toString();
final UUID orgId = orgEntity.getUuid();
-
++
for (UserInfo user : managementService.getAdminUsersForOrganization( orgEntity.getUuid() )) {
try {
Entity admin = managementService.getAdminUserEntityByUuid( user.getUuid() );
@@@ -297,10 -332,34 +326,34 @@@
AdminUserWriteTask task = new AdminUserWriteTask();
task.adminUser = entity;
- addDictionariesToTask( task, entity );
+ addDictionariesToTask( task, entity );
addOrganizationsToTask( task );
- writeQueue.add( task );
+ String actionTaken = "Processed";
+
+ if (ignoreInvalidUsers && (task.orgNamesByUuid.isEmpty()
+ || task.dictionariesByName.isEmpty()
+ || task.dictionariesByName.get( "credentials" ).isEmpty())) {
-
++
+ actionTaken = "Ignored";
-
++
+ } else {
+ writeQueue.add( task );
+ }
+
- Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty() ?
++ Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty() ?
+ 0 : task.dictionariesByName.get( "credentials" ));
-
++
+ logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}",
+ new Object[]{
+ actionTaken,
+ task.adminUser.getProperty( "username" ),
+ task.adminUser.getProperty( "email" ),
+ task.adminUser.getUuid(),
+ task.orgNamesByUuid.size(),
+ task.dictionariesByName.size(),
+ creds == null ? 0 : creds.size()
- } );
++ } );
} catch ( Exception e ) {
logger.error("Error reading data for user " + uuid, e );
@@@ -344,22 -391,17 +385,32 @@@
task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() );
++<<<<<<< HEAD
+ List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
+
++=======
+ List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
-
++
++>>>>>>> master
if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {
-
++
+ // list of orgs from getOrganizationsForAdminUser() is less than expected, use userToOrgsMap
BiMap<UUID, String> bimap = HashBiMap.create();
for (Org org : orgs) {
bimap.put( org.orgId, org.orgName );
}
task.orgNamesByUuid = bimap;
}
++<<<<<<< HEAD
+
+ if ( task.orgNamesByUuid.isEmpty() ) {
+ logger.error("{}:{}:{} has no orgs", new Object[] {
+ task.adminUser.getProperty("username"),
+ task.adminUser.getProperty("email"),
+ task.adminUser.getUuid() } );
+ }
++=======
++>>>>>>> master
}
}
@@@ -485,6 -527,10 +536,10 @@@
jg.writeObject( orgs.get( uuid ) );
jg.writeEndObject();
-
++
+ synchronized (orgsWritten) {
+ orgsWritten.add( uuid );
+ }
}
jg.writeEndArray();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
index 7f8dd1b,3c427e1..c97dc9c
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
@@@ -50,8 -50,8 +50,8 @@@ import static org.apache.usergrid.utils
/**
- * Base class for Usergrid Tools commands. Any class that implements this can be called with
- * Base class for Usergrid Tools commands. Any class that implements this can be called with java -jar {jarname}
- * org.apache.usergrid.tools.{classname}.
++ * Base class for Usergrid Tools commands. Any class that implements this can be called with
+ * java -jar {jarname} org.apache.usergrid.tools.{classname}.
*/
public abstract class ToolBase {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/tools/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --cc stack/tools/src/main/resources/log4j.properties
index 00834cf,def47b4..18ebcc4
--- a/stack/tools/src/main/resources/log4j.properties
+++ b/stack/tools/src/main/resources/log4j.properties
@@@ -26,23 -26,8 +26,28 @@@ log4j.appender.stdout=org.apache.log4j.
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
++<<<<<<< HEAD
+log4j.category.org.apache.usergrid.tools=TRACE
+log4j.category.org.apache.usergrid=ERROR
+
+log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.DaoUtils=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.ConnectionRefImpl=WARN, stdout
+log4j.logger.me.prettyprint.cassandra.hector.TimingLogger=WARN, stdout
+log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
+log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
+#log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
+#log4j.logger.org.apache.usergrid.persistence.hector.CountingMutator=INFO, stdout
+
+log4j.logger.org.apache.usergrid.management.cassandra=DEBUB
+log4j.logger.org.apache.usergrid.tools=INFO
++=======
+ log4j.logger.org.apache.usergrid=INFO
+ log4j.logger.org.apache.usergrid.tools=DEBUG
++>>>>>>> master
log4j.logger.org.apache.usergrid.management.cassandra=WARN
log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN
[3/3] incubator-usergrid git commit: Tools now able to compile again.
Posted by sn...@apache.org.
Tools now able to compile again.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b5144d05
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b5144d05
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b5144d05
Branch: refs/heads/USERGRID-872
Commit: b5144d058189c37a639fa29d67056c0c96ff8531
Parents: fedf165
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jul 22 11:16:33 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jul 22 11:16:33 2015 -0400
----------------------------------------------------------------------
.../usergrid/persistence/EntityManager.java | 4 +-
stack/pom.xml | 2 +-
.../usergrid/services/AbstractService.java | 270 ++++++++++---------
stack/tools/pom.xml | 4 +-
.../java/org/apache/usergrid/tools/Cli.java | 35 +--
.../org/apache/usergrid/tools/DupOrgRepair.java | 2 +-
.../org/apache/usergrid/tools/ExportAdmins.java | 34 +--
.../org/apache/usergrid/tools/ExportApp.java | 92 +++----
.../usergrid/tools/ExportDataCreator.java | 7 +-
.../org/apache/usergrid/tools/ImportAdmins.java | 104 +++----
.../java/org/apache/usergrid/tools/Metrics.java | 2 +-
.../usergrid/tools/OrganizationExport.java | 2 +-
.../apache/usergrid/tools/PopulateSample.java | 2 +-
.../org/apache/usergrid/tools/UserManager.java | 3 +-
.../apache/usergrid/tools/WarehouseExport.java | 117 +++-----
.../apache/usergrid/tools/bean/MetricQuery.java | 6 +-
.../apache/usergrid/tools/ExportAppTest.java | 39 +--
.../usergrid/tools/ExportImportAdminsTest.java | 26 +-
18 files changed, 346 insertions(+), 405 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index c4a77ac..c544914 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -463,14 +463,14 @@ public interface EntityManager {
*
* @param entityRef an entity reference
* @param connectionType type of connection or null.
- * @param connectedEntityType type of entity or null.
+ * @param targetEntityType type of entity or null.
*
* @return a list of connected entity ids.
*
* @throws Exception the exception
*/
public Results getTargetEntities(EntityRef entityRef, String connectionType,
- String connectedEntityType, Level resultsLevel) throws Exception;
+ String targetEntityType, Level resultsLevel) throws Exception;
/**
* Gets the entities connecting to this entity, optionally with the specified connection
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 6f984e3..c03e52f 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -190,8 +190,8 @@
<module>core</module>
<module>services</module>
<module>rest</module>
- <!--
<module>tools</module>
+ <!--
<module>websocket</module>
-->
<!--
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java
index edf8ab2..1c04da9 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractService.java
@@ -26,6 +26,8 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import com.codahale.metrics.Timer;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
@@ -98,8 +100,11 @@ public abstract class AbstractService implements Service {
private Scheduler rxScheduler;
private RxSchedulerFig rxSchedulerFig;
-
-
+ private MetricsFactory metricsFactory;
+ private Timer entityGetTimer;
+ private Timer entitiesGetTimer;
+ private Timer entitiesParallelGetTimer;
+ private Timer invokeTimer;
public AbstractService() {
@@ -112,7 +117,12 @@ public abstract class AbstractService implements Service {
em = sm.getEntityManager();
final Injector injector = sm.getApplicationContext().getBean( Injector.class );
rxScheduler = injector.getInstance( RxTaskScheduler.class ).getAsyncIOScheduler();
- rxSchedulerFig = injector.getInstance( RxSchedulerFig.class );
+ rxSchedulerFig = injector.getInstance(RxSchedulerFig.class);
+ metricsFactory = injector.getInstance(MetricsFactory.class);
+ this.entityGetTimer = metricsFactory.getTimer(this.getClass(), "importEntity.get");
+ this.entitiesGetTimer = metricsFactory.getTimer(this.getClass(), "importEntities.get");
+ this.entitiesParallelGetTimer = metricsFactory.getTimer( this.getClass(),"importEntitiesP.get" );
+ this.invokeTimer = metricsFactory.getTimer( this.getClass(),"service.invoke" );
}
@@ -170,9 +180,9 @@ public abstract class AbstractService implements Service {
return false;
}
if ( "application".equals( context.getOwner().getType() ) ) {
- return Schema.getDefaultSchema().isCollectionReversed( "application", pluralize( info.getItemType() ) );
+ return Schema.getDefaultSchema().isCollectionReversed("application", pluralize(info.getItemType()));
}
- return Schema.getDefaultSchema().isCollectionReversed( info.getContainerType(), info.getCollectionName() );
+ return Schema.getDefaultSchema().isCollectionReversed(info.getContainerType(), info.getCollectionName());
}
@@ -181,9 +191,9 @@ public abstract class AbstractService implements Service {
return null;
}
if ( "application".equals( context.getOwner().getType() ) ) {
- return Schema.getDefaultSchema().getCollectionSort( "application", pluralize( info.getItemType() ) );
+ return Schema.getDefaultSchema().getCollectionSort("application", pluralize(info.getItemType()));
}
- return Schema.getDefaultSchema().getCollectionSort( info.getContainerType(), info.getCollectionName() );
+ return Schema.getDefaultSchema().getCollectionSort(info.getContainerType(), info.getCollectionName());
}
@@ -191,7 +201,7 @@ public abstract class AbstractService implements Service {
if ( privateConnections == null ) {
privateConnections = new LinkedHashSet<String>();
}
- privateConnections.add( connection );
+ privateConnections.add(connection);
}
@@ -199,7 +209,7 @@ public abstract class AbstractService implements Service {
if ( privateConnections == null ) {
privateConnections = new LinkedHashSet<String>();
}
- privateConnections.addAll( connections );
+ privateConnections.addAll(connections);
}
@@ -207,7 +217,7 @@ public abstract class AbstractService implements Service {
if ( declaredConnections == null ) {
declaredConnections = new LinkedHashSet<String>();
}
- declaredConnections.add( connection );
+ declaredConnections.add(connection);
}
@@ -215,7 +225,7 @@ public abstract class AbstractService implements Service {
if ( declaredConnections == null ) {
declaredConnections = new LinkedHashSet<String>();
}
- declaredConnections.addAll( connections );
+ declaredConnections.addAll(connections);
}
@@ -223,7 +233,7 @@ public abstract class AbstractService implements Service {
if ( privateCollections == null ) {
privateCollections = new LinkedHashSet<String>();
}
- privateCollections.add( collection );
+ privateCollections.add(collection);
}
@@ -231,7 +241,7 @@ public abstract class AbstractService implements Service {
if ( privateCollections == null ) {
privateCollections = new LinkedHashSet<String>();
}
- privateCollections.addAll( collections );
+ privateCollections.addAll(collections);
}
@@ -239,7 +249,7 @@ public abstract class AbstractService implements Service {
if ( declaredCollections == null ) {
declaredCollections = new LinkedHashSet<String>();
}
- declaredCollections.add( collection );
+ declaredCollections.add(collection);
}
@@ -247,7 +257,7 @@ public abstract class AbstractService implements Service {
if ( declaredCollections == null ) {
declaredCollections = new LinkedHashSet<String>();
}
- declaredCollections.addAll( collections );
+ declaredCollections.addAll(collections);
}
@@ -255,7 +265,7 @@ public abstract class AbstractService implements Service {
if ( replaceParameters == null ) {
replaceParameters = new LinkedHashMap<List<String>, List<String>>();
}
- replaceParameters.put( find, replace );
+ replaceParameters.put(find, replace);
}
@@ -263,7 +273,7 @@ public abstract class AbstractService implements Service {
if ( serviceCommands == null ) {
serviceCommands = new LinkedHashSet<String>();
}
- serviceCommands.add( command );
+ serviceCommands.add(command);
}
@@ -271,7 +281,7 @@ public abstract class AbstractService implements Service {
if ( serviceCommands == null ) {
serviceCommands = new LinkedHashSet<String>();
}
- serviceCommands.addAll( commands );
+ serviceCommands.addAll(commands);
}
@@ -279,7 +289,7 @@ public abstract class AbstractService implements Service {
if ( entityDictionaries == null ) {
entityDictionaries = new LinkedHashSet<EntityDictionaryEntry>();
}
- entityDictionaries.add( dictionary );
+ entityDictionaries.add(dictionary);
}
@@ -305,7 +315,7 @@ public abstract class AbstractService implements Service {
if ( metadataTypes == null ) {
metadataTypes = new LinkedHashSet<String>();
}
- metadataTypes.add( type );
+ metadataTypes.add(type);
}
@@ -313,7 +323,7 @@ public abstract class AbstractService implements Service {
if ( metadataTypes == null ) {
metadataTypes = new LinkedHashSet<String>();
}
- metadataTypes.addAll( typeList );
+ metadataTypes.addAll(typeList);
}
@@ -321,7 +331,7 @@ public abstract class AbstractService implements Service {
if ( entityCommands == null ) {
entityCommands = new LinkedHashSet<String>();
}
- entityCommands.add( command );
+ entityCommands.add(command);
}
@@ -329,7 +339,7 @@ public abstract class AbstractService implements Service {
if ( entityCommands == null ) {
entityCommands = new LinkedHashSet<String>();
}
- entityCommands.addAll( commands );
+ entityCommands.addAll(commands);
}
@@ -354,69 +364,79 @@ public abstract class AbstractService implements Service {
@Override
public Entity importEntity( ServiceRequest request, Entity entity ) throws Exception {
- if ( entity == null ) {
- return null;
- }
+ Timer.Context getEntityTimer = entityGetTimer.time();
+ try {
+ if (entity == null) {
+ return null;
+ }
- if ( !isRootService() ) {
- return sm.importEntity( request, entity );
- }
+ if (!isRootService()) {
+ return sm.importEntity(request, entity);
+ }
- String path = request.getPath() + "/" + entity.getUuid();
- Map<String, Object> metadata = new LinkedHashMap<String, Object>();
- metadata.put( "path", path );
- if ( defaultEntityMetadata != null ) {
- metadata.putAll( defaultEntityMetadata );
- }
+ String path = request.getPath() + "/" + entity.getUuid();
+ Map<String, Object> metadata = new LinkedHashMap<String, Object>();
+ metadata.put("path", path);
- Set<Object> connections = getConnectedTypesSet( entity );
- if ( connections != null ) {
- Map<String, Object> m = new LinkedHashMap<String, Object>();
- for ( Object n : connections ) {
- m.put( n.toString(), path + "/" + n );
+ if (defaultEntityMetadata != null) {
+ metadata.putAll(defaultEntityMetadata);
+ }
+
+ Set<Object> connections = getConnectedTypesSet(entity);
+ if (connections != null) {
+ Map<String, Object> m = new LinkedHashMap<String, Object>();
+ for (Object n : connections) {
+ m.put(n.toString(), path + "/" + n);
+ }
+ metadata.put("connections", m);
}
- metadata.put( "connections", m );
- }
- Set<Object> connecting = getConnectingTypesSet( entity );
- if ( connecting != null ) {
- Map<String, Object> m = new LinkedHashMap<String, Object>();
- for ( Object n : connecting ) {
- m.put( n.toString(), path + "/connecting/" + n );
+ Set<Object> connecting = getConnectingTypesSet(entity);
+ if (connecting != null) {
+ Map<String, Object> m = new LinkedHashMap<String, Object>();
+ for (Object n : connecting) {
+ m.put(n.toString(), path + "/connecting/" + n);
+ }
+ metadata.put("connecting", m);
}
- metadata.put( "connecting", m );
- }
- Set<String> collections = getCollectionSet( entity );
- if ( collections != null ) {
- Map<String, Object> m = new LinkedHashMap<String, Object>();
- for ( Object n : collections ) {
- m.put( n.toString(), path + "/" + n );
+ Set<String> collections = getCollectionSet(entity);
+ if (collections != null) {
+ Map<String, Object> m = new LinkedHashMap<String, Object>();
+ for (Object n : collections) {
+ m.put(n.toString(), path + "/" + n);
+ }
+ metadata.put("collections", m);
}
- metadata.put( "collections", m );
- }
- if ( entityDictionaries != null ) {
- Map<String, Object> m = new LinkedHashMap<String, Object>();
- for ( EntityDictionaryEntry dict : entityDictionaries ) {
- m.put( dict.getName(), path + "/" + dict.getPath() );
+ if (entityDictionaries != null) {
+ Map<String, Object> m = new LinkedHashMap<String, Object>();
+ for (EntityDictionaryEntry dict : entityDictionaries) {
+ m.put(dict.getName(), path + "/" + dict.getPath());
+ }
+ metadata.put("sets", m);
}
- metadata.put( "sets", m );
- }
- if ( metadata.size() > 0 ) {
- entity.mergeMetadata( metadata );
+ if (metadata.size() > 0) {
+ entity.mergeMetadata(metadata);
+ }
+ return entity;
+ }finally {
+ getEntityTimer.stop();
}
- return entity;
}
public void importEntities( ServiceRequest request, Results results ) throws Exception {
-
- List<Entity> entities = results.getEntities();
- if ( entities != null ) {
- importEntitiesParallel(request, results);
+ Timer.Context timer = entitiesGetTimer.time();
+ try {
+ List<Entity> entities = results.getEntities();
+ if (entities != null) {
+ importEntitiesParallel(request, results);
+ }
+ }finally {
+ timer.stop();
}
}
@@ -427,42 +447,45 @@ public abstract class AbstractService implements Service {
* @param results
*/
private void importEntitiesParallel(final ServiceRequest request, final Results results ) {
+ Timer.Context timer = entitiesParallelGetTimer.time();
+ try {
+ //create our tuples
+ final Observable<EntityTuple> tuples = Observable.create(new Observable.OnSubscribe<EntityTuple>() {
+ @Override
+ public void call(final Subscriber<? super EntityTuple> subscriber) {
+ subscriber.onStart();
+
+ final List<Entity> entities = results.getEntities();
+ final int size = entities.size();
+ for (int i = 0; i < size && !subscriber.isUnsubscribed(); i++) {
+ subscriber.onNext(new EntityTuple(i, entities.get(i)));
+ }
- //create our tuples
- final Observable<EntityTuple> tuples = Observable.create( new Observable.OnSubscribe<EntityTuple>() {
- @Override
- public void call( final Subscriber<? super EntityTuple> subscriber ) {
- subscriber.onStart();
-
- final List<Entity> entities = results.getEntities();
- final int size = entities.size();
- for ( int i = 0; i < size && !subscriber.isUnsubscribed(); i++ ) {
- subscriber.onNext( new EntityTuple( i, entities.get( i ) ) );
+ subscriber.onCompleted();
}
+ });
- subscriber.onCompleted();
- }
- } );
-
- //now process them in parallel up to 10 threads
+ //now process them in parallel up to 10 threads
- tuples.flatMap( tuple -> {
- //map the entity into the tuple
- return Observable.just( tuple ).doOnNext( parallelTuple -> {
- //import the entity and set it at index
- try {
+ tuples.flatMap(tuple -> {
+ //map the entity into the tuple
+ return Observable.just(tuple).doOnNext(parallelTuple -> {
+ //import the entity and set it at index
+ try {
- final Entity imported = importEntity( request, parallelTuple.entity );
+ final Entity imported = importEntity(request, parallelTuple.entity);
- if(imported != null) {
- results.setEntity( parallelTuple.index, imported );
+ if (imported != null) {
+ results.setEntity(parallelTuple.index, imported);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
- }
- catch ( Exception e ) {
- throw new RuntimeException(e);
- }
- } ).subscribeOn( rxScheduler );
- }, rxSchedulerFig.getImportThreads() ).toBlocking().lastOrDefault( null );
+ }).subscribeOn(rxScheduler);
+ }, rxSchedulerFig.getImportThreads()).toBlocking().lastOrDefault(null);
+ } finally {
+ timer.stop();
+ }
}
@@ -528,7 +551,7 @@ public abstract class AbstractService implements Service {
entity.addProperties( payload.getProperties() );
return entity;
}
- logger.error( "Attempted update of entity reference rather than full entity, currently unsupport - MUSTFIX" );
+ logger.error("Attempted update of entity reference rather than full entity, currently unsupport - MUSTFIX");
throw new NotImplementedException();
}
@@ -555,7 +578,7 @@ public abstract class AbstractService implements Service {
public Set<Object> getConnectedTypesSet( EntityRef ref ) throws Exception {
- final Set<String> connections = em.getConnectionsAsSource( ref );
+ final Set<String> connections = em.getConnectionsAsSource(ref);
if ( connections == null ) {
return null;
@@ -574,7 +597,7 @@ public abstract class AbstractService implements Service {
public Set<Object> getConnectingTypesSet( EntityRef ref ) throws Exception {
- final Set<String> connections = em.getConnectionsAsTarget( ref );
+ final Set<String> connections = em.getConnectionsAsTarget(ref);
if ( connections == null ) {
return null;
@@ -593,7 +616,7 @@ public abstract class AbstractService implements Service {
public Set<String> getCollectionSet( EntityRef ref ) {
- Set<String> set = Schema.getDefaultSchema().getCollectionNames( ref.getType() );
+ Set<String> set = Schema.getDefaultSchema().getCollectionNames(ref.getType());
set = new LinkedHashSet<String>( set );
if ( declaredCollections != null ) {
set.addAll( declaredCollections );
@@ -612,7 +635,7 @@ public abstract class AbstractService implements Service {
public ServiceResults invoke( ServiceAction action, ServiceRequest request, ServiceResults previousResults,
ServicePayload payload ) throws Exception {
- ServiceContext context = getContext( action, request, previousResults, payload );
+ ServiceContext context = getContext(action, request, previousResults, payload);
return invoke( context );
}
@@ -629,7 +652,7 @@ public abstract class AbstractService implements Service {
EntityRef owner = request.getOwner();
String collectionName =
"application".equals( owner.getType() ) ? pluralize( info.getItemType() ) : info.getCollectionName();
- List<ServiceParameter> parameters = filter( request.getParameters(), replaceParameters );
+ List<ServiceParameter> parameters = filter(request.getParameters(), replaceParameters);
ServiceParameter first_parameter = null;
if ( !isEmpty( parameters ) ) {
@@ -652,7 +675,7 @@ public abstract class AbstractService implements Service {
if ( first_parameter instanceof QueryParameter ) {
query = first_parameter.getQuery();
}
- parameters = mergeQueries( query, parameters );
+ parameters = mergeQueries(query, parameters);
if ( first_parameter instanceof IdParameter ) {
UUID id = first_parameter.getId();
@@ -678,9 +701,8 @@ public abstract class AbstractService implements Service {
public ServiceResults invoke( ServiceContext context ) throws Exception {
-
ServiceResults results = null;
-
+ Timer.Context time = invokeTimer.time();
String metadataType = checkForServiceMetadata( context );
if ( metadataType != null ) {
return handleServiceMetadata( context, metadataType );
@@ -708,8 +730,9 @@ public abstract class AbstractService implements Service {
}
results = handleEntityDictionary( context, results, entityDictionary );
- results = handleEntityCommand( context, results, entityCommand );
+ results = handleEntityCommand(context, results, entityCommand);
+ time.stop();
return results;
}
@@ -718,19 +741,19 @@ public abstract class AbstractService implements Service {
switch ( context.getAction() ) {
case GET:
- return getItemById( context, id );
+ return getItemById(context, id);
case POST:
- return postItemById( context, id );
+ return postItemById(context, id);
case PUT:
- return putItemById( context, id );
+ return putItemById(context, id);
case DELETE:
- return deleteItemById( context, id );
+ return deleteItemById(context, id);
case HEAD:
- return headItemById( context, id );
+ return headItemById(context, id);
}
throw new ServiceInvocationException( context, "Request action unhandled " + context.getAction() );
@@ -738,25 +761,20 @@ public abstract class AbstractService implements Service {
public ServiceResults invokeItemWithName( ServiceContext context, String name ) throws Exception {
-
- switch ( context.getAction() ) {
+ switch (context.getAction()) {
case GET:
- return getItemByName( context, name );
-
+ return getItemByName(context, name);
case POST:
- return postItemByName( context, name );
-
+ return postItemByName(context, name);
case PUT:
- return putItemByName( context, name );
-
+ return putItemByName(context, name);
case DELETE:
- return deleteItemByName( context, name );
-
+ return deleteItemByName(context, name);
case HEAD:
- return headItemByName( context, name );
+ return headItemByName(context, name);
+ default:
+ throw new ServiceInvocationException(context, "Request action unhandled " + context.getAction());
}
-
- throw new ServiceInvocationException( context, "Request action unhandled " + context.getAction() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/pom.xml
----------------------------------------------------------------------
diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml
index d7bfc98..6e0dc50 100644
--- a/stack/tools/pom.xml
+++ b/stack/tools/pom.xml
@@ -21,7 +21,7 @@
<parent>
<groupId>org.apache.usergrid</groupId>
<artifactId>usergrid</artifactId>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.1.0-SNAPSHOT</version>
<relativePath>../</relativePath>
</parent>
@@ -29,7 +29,7 @@
<name>Usergrid Tools</name>
<description>Command line tools for Usergrid system.</description>
<packaging>jar</packaging>
- <version>2.0.0-SNAPSHOT</version>
+ <version>2.1.0-SNAPSHOT</version>
<reporting>
<plugins>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java
index 9594403..511dcaa 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Cli.java
@@ -17,6 +17,19 @@
package org.apache.usergrid.tools;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.services.*;
+import org.apache.usergrid.utils.HttpUtils;
+import org.apache.usergrid.utils.JsonUtils;
+import org.apache.usergrid.utils.UUIDUtils;
+import org.codehaus.jackson.JsonFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -25,26 +38,6 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
-import org.codehaus.jackson.JsonFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.services.ServiceAction;
-import org.apache.usergrid.services.ServiceManager;
-import org.apache.usergrid.services.ServiceParameter;
-import org.apache.usergrid.services.ServicePayload;
-import org.apache.usergrid.services.ServiceRequest;
-import org.apache.usergrid.services.ServiceResults;
-import org.apache.usergrid.utils.HttpUtils;
-import org.apache.usergrid.utils.JsonUtils;
-import org.apache.usergrid.utils.UUIDUtils;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
-
public class Cli extends ToolBase {
@@ -79,7 +72,7 @@ public class Cli extends ToolBase {
}
- public void handleInput() throws QueryParseException {
+ public void handleInput() throws Exception {
BufferedReader d = new BufferedReader( new InputStreamReader( System.in ) );
UUID applicationId = null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java b/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java
index a20dda1..ed4deee 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/DupOrgRepair.java
@@ -238,7 +238,7 @@ public class DupOrgRepair extends ExportingToolBase {
app.getKey(), app.getValue(), targetOrgId
} );
- managementService.addApplicationToOrganization( targetOrgId, app.getValue(), appEntity);
+ managementService.addApplicationToOrganization( targetOrgId, appEntity);
}
// now delete the original org
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index 0bb74ab..bb6e863 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@ -23,8 +23,10 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.utils.StringUtils;
import org.codehaus.jackson.JsonGenerator;
@@ -345,15 +347,15 @@ public class ExportAdmins extends ExportingToolBase {
0 : task.dictionariesByName.get( "credentials" ));
logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}",
- new Object[]{
- actionTaken,
- task.adminUser.getProperty( "username" ),
- task.adminUser.getProperty( "email" ),
- task.adminUser.getUuid(),
- task.orgNamesByUuid.size(),
- task.dictionariesByName.size(),
- creds == null ? 0 : creds.size()
- } );
+ new Object[]{
+ actionTaken,
+ task.adminUser.getProperty( "username" ),
+ task.adminUser.getProperty( "email" ),
+ task.adminUser.getUuid(),
+ task.orgNamesByUuid.size(),
+ task.dictionariesByName.size(),
+ creds == null ? 0 : creds.size()
+ } );
} catch ( Exception e ) {
logger.error("Error reading data for user " + uuid, e );
@@ -385,13 +387,9 @@ public class ExportAdmins extends ExportingToolBase {
task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() );
-<<<<<<< HEAD
- List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
-=======
List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
->>>>>>> master
if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {
// list of orgs from getOrganizationsForAdminUser() is less than expected, use userToOrgsMap
@@ -401,16 +399,6 @@ public class ExportAdmins extends ExportingToolBase {
}
task.orgNamesByUuid = bimap;
}
-<<<<<<< HEAD
-
- if ( task.orgNamesByUuid.isEmpty() ) {
- logger.error("{}:{}:{} has no orgs", new Object[] {
- task.adminUser.getProperty("username"),
- task.adminUser.getProperty("email"),
- task.adminUser.getUuid() } );
- }
-=======
->>>>>>> master
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
index db975e6..f5e5d34 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportApp.java
@@ -49,13 +49,13 @@ import java.util.concurrent.atomic.AtomicInteger;
/**
* Export all entities and connections of a Usergrid app.
- *
+ *
* Exports data files to specified directory.
- *
+ *
* Will create as many output files as there are writeThreads (by default: 10).
- *
+ *
* Will create two types of files: *.entities for Usegrird entities and *.collections for entity to entity connections.
- *
+ *
* Every line of the data files is a complete JSON object.
*/
public class ExportApp extends ExportingToolBase {
@@ -63,7 +63,7 @@ public class ExportApp extends ExportingToolBase {
static final String APPLICATION_NAME = "application";
private static final String WRITE_THREAD_COUNT = "writeThreads";
-
+
String applicationName;
String organizationName;
@@ -76,7 +76,7 @@ public class ExportApp extends ExportingToolBase {
Map<Thread, JsonGenerator> entityGeneratorsByThread = new HashMap<Thread, JsonGenerator>();
Map<Thread, JsonGenerator> connectionGeneratorsByThread = new HashMap<Thread, JsonGenerator>();
- int writeThreadCount = 10; // set via CLI option; limiting write will limit output files
+ int writeThreadCount = 10; // set via CLI option; limiting write will limit output files
@Override
@@ -96,9 +96,9 @@ public class ExportApp extends ExportingToolBase {
return options;
}
-
+
/**
- * Tool entry point.
+ * Tool entry point.
*/
@Override
public void runTool(CommandLine line) throws Exception {
@@ -123,7 +123,7 @@ public class ExportApp extends ExportingToolBase {
startSpring();
- UUID applicationId = emf.lookupApplication( applicationName );
+ UUID applicationId = emf.lookupApplication( applicationName ).get();
if (applicationId == null) {
throw new RuntimeException( "Cannot find application " + applicationName );
}
@@ -134,7 +134,7 @@ public class ExportApp extends ExportingToolBase {
writeScheduler = Schedulers.from( writeThreadPoolExecutor );
Observable<String> collectionsObservable = Observable.create( new CollectionsObservable( em ) );
-
+
collectionsObservable.flatMap( new Func1<String, Observable<ExportEntity>>() {
public Observable<ExportEntity> call(String collection) {
@@ -155,24 +155,24 @@ public class ExportApp extends ExportingToolBase {
.doOnCompleted( new FileWrapUpAction() )
.toBlocking().last();
}
-
-
+
+
// ----------------------------------------------------------------------------------------
// reading data
-
+
/**
* Emits collection names found in application.
*/
class CollectionsObservable implements rx.Observable.OnSubscribe<String> {
EntityManager em;
-
+
public CollectionsObservable(EntityManager em) {
this.em = em;
}
public void call(Subscriber<? super String> subscriber) {
-
+
int count = 0;
try {
Map<String, Object> collectionMetadata = em.getApplicationCollectionMetadata();
@@ -180,17 +180,17 @@ public class ExportApp extends ExportingToolBase {
subscriber.onNext( collection );
count++;
}
-
+
} catch (Exception e) {
subscriber.onError( e );
}
-
+
subscriber.onCompleted();
logger.info( "Completed. Read {} collection names", count );
}
}
-
+
/**
* Emits entities of collection.
*/
@@ -206,9 +206,9 @@ public class ExportApp extends ExportingToolBase {
public void call(Subscriber<? super ExportEntity> subscriber) {
logger.info("Starting to read entities of collection {}", collection);
-
+
subscriber.onStart();
-
+
try {
int count = 0;
@@ -229,16 +229,16 @@ public class ExportApp extends ExportingToolBase {
}
dictionariesByName.put( dictionary, dict );
}
-
- ExportEntity exportEntity = new ExportEntity(
- organizationName,
- applicationName,
- entity,
+
+ ExportEntity exportEntity = new ExportEntity(
+ organizationName,
+ applicationName,
+ entity,
dictionariesByName );
-
+
subscriber.onNext( exportEntity );
count++;
-
+
} catch (Exception e) {
logger.error("Error reading entity " + entity.getUuid() +" from collection " + collection);
}
@@ -252,14 +252,14 @@ public class ExportApp extends ExportingToolBase {
subscriber.onCompleted();
logger.info("Completed collection {}. Read {} entities", collection, count);
-
+
} catch ( Exception e ) {
subscriber.onError(e);
}
}
}
-
+
/**
* Emits connections of an entity.
*/
@@ -276,51 +276,51 @@ public class ExportApp extends ExportingToolBase {
logger.info( "Starting to read connections for entity {} type {}",
exportEntity.getEntity().getName(), exportEntity.getEntity().getType() );
-
+
int count = 0;
-
+
try {
Set<String> connectionTypes = em.getConnectionTypes( exportEntity.getEntity() );
for (String connectionType : connectionTypes) {
- Results results = em.getConnectedEntities(
- exportEntity.getEntity().getUuid(), connectionType, null, Results.Level.CORE_PROPERTIES );
+ Results results = em.getTargetEntities(
+ exportEntity.getEntity(), connectionType, null, Query.Level.CORE_PROPERTIES );
for (Entity connectedEntity : results.getEntities()) {
try {
-
- ExportConnection connection = new ExportConnection(
+
+ ExportConnection connection = new ExportConnection(
applicationName,
organizationName,
- connectionType,
- exportEntity.getEntity().getUuid(),
+ connectionType,
+ exportEntity.getEntity().getUuid(),
connectedEntity.getUuid());
-
+
subscriber.onNext( connection );
count++;
} catch (Exception e) {
- logger.error( "Error reading connection entity "
+ logger.error( "Error reading connection entity "
+ exportEntity.getEntity().getUuid() + " -> " + connectedEntity.getType());
}
}
}
-
+
} catch (Exception e) {
subscriber.onError( e );
}
-
+
subscriber.onCompleted();
logger.info("Completed entity {} type {} connections count {}",
new Object[] { exportEntity.getEntity().getName(), exportEntity.getEntity().getType(), count });
}
}
-
+
// ----------------------------------------------------------------------------------------
// writing data
-
-
+
+
/**
* Writes entities to JSON file.
*/
@@ -358,7 +358,7 @@ public class ExportApp extends ExportingToolBase {
}
}
-
+
/**
* Writes connection to JSON file.
*/
@@ -396,7 +396,7 @@ public class ExportApp extends ExportingToolBase {
}
}
-
+
private class FileWrapUpAction implements Action0 {
@Override
public void call() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java
index 6fa4896..4b9e5a0 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportDataCreator.java
@@ -16,7 +16,6 @@
*/
package org.apache.usergrid.tools;
-
import io.codearte.jfairy.Fairy;
import io.codearte.jfairy.producer.company.Company;
import io.codearte.jfairy.producer.person.Person;
@@ -189,7 +188,7 @@ public class ExportDataCreator extends ToolBase {
try {
EntityRef ref = em.getAlias( "company", company.name() );
Entity companyEntity = (ref == null) ? null : em.get( ref );
-
+
// create company if it does not exist yet
if ( companyEntity == null ) {
final Map<String, Object> companyMap = new HashMap<String, Object>() {{
@@ -213,7 +212,7 @@ public class ExportDataCreator extends ToolBase {
logger.error("Error creating or connecting company", e);
continue;
}
-
+
try {
for (int j = 0; j < 5; j++) {
Activity activity = new Activity();
@@ -236,7 +235,7 @@ public class ExportDataCreator extends ToolBase {
}
}
}
-
+
} catch (Exception e) {
logger.error("Error creating activities", e);
continue;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
index f39ef9b..c8c4d99 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ImportAdmins.java
@@ -17,17 +17,16 @@
package org.apache.usergrid.tools;
-import com.sun.org.apache.bcel.internal.generic.DUP;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.io.filefilter.PrefixFileFilter;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Identifier;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.entities.User;
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
@@ -39,7 +38,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.util.*;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -47,18 +49,18 @@ import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import org.apache.usergrid.persistence.index.query.Identifier;
/**
* Import Admin Users and metadata including organizations and passwords.
- *
- * Usage Example:
- *
+ *
+ * Usage Example:
+ *
* java -Xmx8000m -Dlog4j.configuration=file:/home/me/log4j.properties -classpath . \
* -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 \
- * -host casshost -inputDir=/home/me/import-data
- *
+ * -host casshost -inputDir=/home/me/import-data
+ *
* If you want to provide any property overrides, put properties file named usergrid-custom-tools.properties
* in the same directory where you run the above command. For example, you might want to set the Cassandra
* client threads and import to a specific set of keyspaces:
@@ -97,8 +99,8 @@ public class ImportAdmins extends ToolBase {
AtomicInteger writeEmptyCount = new AtomicInteger( 0 );
AtomicInteger auditEmptyCount = new AtomicInteger( 0 );
AtomicInteger metadataEmptyCount = new AtomicInteger( 0 );
-
-
+
+
static class DuplicateUser {
String email;
String username;
@@ -110,7 +112,7 @@ public class ImportAdmins extends ToolBase {
}
}
}
-
+
@Override
@@ -269,7 +271,7 @@ public class ImportAdmins extends ToolBase {
long timeRemaining = (long) ( sizeLast / (rateAverageSum / iterations) );
logger.info("++PROGRESS ({}): sizeLast={} nowSize={} processed={} rateLast={}/s rateAvg={}/s timeRemaining={}s",
- new Object[] {
+ new Object[] {
identifier, sizeLast, sizeNow, processed, rateLast, (rateAverageSum / iterations), timeRemaining } );
}
@@ -355,7 +357,7 @@ public class ImportAdmins extends ToolBase {
workerThread.start();
metadataWorkerThreadMap.put(worker, workerThread);
}
-
+
logger.info( "Started {} metadata workers", writeThreadCount );
}
@@ -363,7 +365,7 @@ public class ImportAdmins extends ToolBase {
@SuppressWarnings("unchecked")
private void importMetadata(String fileName, int writeThreads) throws Exception {
- EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID);
File metadataFile = new File(importDir, fileName);
@@ -401,10 +403,10 @@ public class ImportAdmins extends ToolBase {
try {
EntityRef entityRef = new SimpleEntityRef( "user", UUID.fromString( entityOwnerId ) );
Map<String, Object> metadata = (Map<String, Object>) jp.readValueAs( Map.class );
-
+
workQueue.put( new ImportMetadataTask( entityRef, metadata ) );
logger.debug( "Put user {} in metadata queue", entityRef.getUuid() );
-
+
} catch ( Exception e ) {
logger.debug( "Error with user {}, not putting in metadata queue", entityOwnerId );
}
@@ -426,7 +428,7 @@ public class ImportAdmins extends ToolBase {
EntityManager em, EntityRef entityRef, Map<String, Object> metadata) throws Exception {
DuplicateUser dup = dupsByDupUuid.get( entityRef.getUuid() );
-
+
if ( dup == null ) { // not a duplicate
User user = em.get( entityRef, User.class );
@@ -476,7 +478,7 @@ public class ImportAdmins extends ToolBase {
}
}
}
-
+
Map<String, Object> dictionariesMap = (Map<String, Object>) metadata.get("dictionaries");
if (dictionariesMap != null && !dictionariesMap.isEmpty()) {
for (String name : dictionariesMap.keySet()) {
@@ -501,17 +503,17 @@ public class ImportAdmins extends ToolBase {
} else {
logger.warn("User {} has no dictionaries", entityRef.getUuid() );
}
-
+
} else { // this is a duplicate user, so merge orgs
logger.info("Processing duplicate username={} email={}", dup.email, dup.username );
-
- Identifier identifier = dup.email != null ?
+
+ Identifier identifier = dup.email != null ?
Identifier.fromEmail( dup.email ) : Identifier.from( dup.username );
User originalUser = em.get( em.getUserByIdentifier(identifier), User.class );
// get map of original user's orgs
-
+
UserInfo originalUserInfo = managementService.getAdminUserByEmail( originalUser.getEmail() );
Map<String, Object> originalUserOrgData =
managementService.getAdminUserOrganizationData( originalUser.getUuid() );
@@ -530,7 +532,7 @@ public class ImportAdmins extends ToolBase {
if (originalUserOrgs.get( orgName ) == null) { // original user does not have this org
OrganizationInfo orgInfo = managementService.getOrganizationByUuid( orgUuid );
-
+
if (orgInfo == null) { // org does not exist yet, create it and add original user to it
try {
managementService.createOrganization( orgUuid, orgName, originalUserInfo, false );
@@ -539,9 +541,9 @@ public class ImportAdmins extends ToolBase {
logger.debug( "Created new org {} for user {}:{}:{} from duplicate user {}:{}",
new Object[]{
orgInfo.getName(),
- originalUser.getUsername(),
+ originalUser.getUsername(),
originalUser.getEmail(),
- originalUser.getUuid(),
+ originalUser.getUuid(),
dup.username, dup.email
});
@@ -554,20 +556,20 @@ public class ImportAdmins extends ToolBase {
logger.debug( "Added to org user {}:{}:{} from duplicate user {}:{}",
new Object[]{
orgInfo.getName(),
- originalUser.getUsername(),
+ originalUser.getUsername(),
originalUser.getEmail(),
- originalUser.getUuid(),
+ originalUser.getUuid(),
dup.username, dup.email
});
} catch (Exception e) {
- logger.error( "Error Adding user {} to org {}",
+ logger.error( "Error Adding user {} to org {}",
new Object[]{originalUserInfo.getEmail(), orgName} );
}
}
-
+
} // else original user already has this org
-
+
}
}
}
@@ -612,7 +614,7 @@ public class ImportAdmins extends ToolBase {
public void run() {
int count = 0;
- EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+ EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
long durationSum = 0;
@@ -647,7 +649,7 @@ public class ImportAdmins extends ToolBase {
durationSum += duration;
//logger.debug( "Audited {}th admin", userCount );
-
+
if ( count % 100 == 0 ) {
logger.info( "Audited {}. Average Audit Rate: {}(ms)", count, durationSum / count );
}
@@ -690,7 +692,7 @@ public class ImportAdmins extends ToolBase {
public void run() {
int count = 0;
- EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+ EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
long durationSum = 0;
@@ -705,20 +707,20 @@ public class ImportAdmins extends ToolBase {
continue;
}
metadataEmptyCount.set( 0 );
-
+
long startTime = System.currentTimeMillis();
-
+
importEntityMetadata( em, task.entityRef, task.metadata );
-
+
long stopTime = System.currentTimeMillis();
long duration = stopTime - startTime;
durationSum += duration;
metadataCount.getAndIncrement();
count++;
-
+
if ( count % 30 == 0 ) {
logger.info( "Imported {} metadata of total {} expected. " +
- "Average metadata Imported Rate: {}(ms)",
+ "Average metadata Imported Rate: {}(ms)",
new Object[] { metadataCount.get(), userCount.get(), durationSum / count });
}
@@ -752,7 +754,7 @@ public class ImportAdmins extends ToolBase {
public void run() {
int count = 0;
- EntityManager em = emf.getEntityManager(MANAGEMENT_APPLICATION_ID);
+ EntityManager em = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
long durationSum = 0;
@@ -776,7 +778,7 @@ public class ImportAdmins extends ToolBase {
try {
long startTime = System.currentTimeMillis();
-
+
em.create(uuid, type, entityProps);
logger.debug( "Imported admin user {}:{}:{}",
@@ -787,20 +789,20 @@ public class ImportAdmins extends ToolBase {
long stopTime = System.currentTimeMillis();
long duration = stopTime - startTime;
durationSum += duration;
-
+
count++;
if (count % 30 == 0) {
logger.info( "This worked has imported {} users of total {} imported so far. " +
- "Average Creation Rate: {}ms",
+ "Average Creation Rate: {}ms",
new Object[] { count, userCount.get(), durationSum / count });
}
-
+
} catch (DuplicateUniquePropertyExistsException de) {
String dupProperty = de.getPropertyName();
handleDuplicateAccount( em, dupProperty, entityProps );
continue;
-
+
} catch (Exception e) {
logger.error("Error", e);
}
@@ -811,19 +813,19 @@ public class ImportAdmins extends ToolBase {
}
}
-
+
private void handleDuplicateAccount(EntityManager em, String dupProperty, Map<String, Object> entityProps ) {
logger.info( "Processing duplicate user {}:{}:{} with duplicate {}", new Object[]{
- entityProps.get( "username" ),
- entityProps.get( "email" ),
- entityProps.get( "uuid" ),
+ entityProps.get( "username" ),
+ entityProps.get( "email" ),
+ entityProps.get( "uuid" ),
dupProperty} );
-
+
UUID dupUuid = UUID.fromString( entityProps.get("uuid").toString() );
try {
dupsByDupUuid.put( dupUuid, new DuplicateUser( dupProperty, entityProps ) );
-
+
} catch (Exception e) {
logger.error("Error processing dup user {}:{}:{}",
new Object[] {entityProps.get( "username" ), entityProps.get("email"), dupUuid});
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java
index b308290..a056f77 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Metrics.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.codehaus.jackson.JsonGenerator;
import org.apache.usergrid.management.ApplicationInfo;
import org.apache.usergrid.management.OrganizationInfo;
@@ -46,7 +47,6 @@ import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.BiMap;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Ordering;
-import org.apache.usergrid.persistence.index.query.CounterResolution;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java
index 4fbccb2..4493b67 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/OrganizationExport.java
@@ -25,7 +25,7 @@ import java.util.UUID;
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
import org.apache.commons.cli.CommandLine;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
index b67e9eb..48e2af6 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/PopulateSample.java
@@ -46,7 +46,7 @@ import static org.apache.usergrid.services.ServicePayload.payload;
public class PopulateSample extends ToolBase {
- private static final Logger logger = LoggerFactory.getLogger( Export.class );
+ private static final Logger logger = LoggerFactory.getLogger( PopulateSample.class );
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java
index 3b5383d..e4195a4 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UserManager.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.tools;
import com.google.common.collect.BiMap;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.management.UserInfo;
import org.apache.commons.cli.CommandLine;
@@ -65,7 +66,7 @@ public class UserManager extends ToolBase {
logger.info( mapToFormattedJsonString( orgs ) );
logger.info("--- User dictionaries:");
- EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
+ EntityManager em = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
User user = em.get( userInfo.getUuid(), User.class );
Set<String> dictionaries = em.getDictionaries( user );
for (String dictionary : dictionaries) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java
index 7b60257..564bd7e 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/WarehouseExport.java
@@ -17,63 +17,42 @@
package org.apache.usergrid.tools;
+import au.com.bytecode.opencsv.CSVReader;
+import au.com.bytecode.opencsv.CSVWriter;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.PutObjectResult;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Module;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.entities.Application;
+import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
+import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.netty.config.NettyPayloadModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.FilenameFilter;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import org.jclouds.ContextBuilder;
-import org.jclouds.blobstore.AsyncBlobStore;
-import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.domain.BlobBuilder;
-import org.jclouds.blobstore.options.PutOptions;
-import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
-import org.jclouds.logging.log4j.config.Log4JLoggingModule;
-import org.jclouds.netty.config.NettyPayloadModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.persistence.entities.Application;
-import org.apache.usergrid.persistence.schema.CollectionInfo;
-import org.apache.usergrid.utils.InflectionUtils;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-
-import com.google.common.collect.BiMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.inject.Module;
-
-import au.com.bytecode.opencsv.CSVReader;
-import au.com.bytecode.opencsv.CSVWriter;
-
import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
-import org.apache.usergrid.persistence.index.query.Query.Level;
/**
@@ -195,40 +174,16 @@ public class WarehouseExport extends ExportingToolBase {
.of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
new NettyPayloadModule() );
- BlobStoreContext context =
- ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES )
- .overrides( overrides ).buildView( BlobStoreContext.class );
-
- // Create Container (the bucket in s3)
- try {
- AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync
- // BlobStore (returns false if it already exists)
- ListenableFuture<Boolean> container = blobStore.createContainerInLocation( null, bucketName );
- if ( container.get() ) {
- LOG.info( "Created bucket " + bucketName );
- }
- }
- catch ( Exception ex ) {
- logger.error( "Could not start binary service: {}", ex.getMessage() );
- throw new RuntimeException( ex );
- }
-
- try {
- File file = new File( fileName );
- AsyncBlobStore blobStore = context.getAsyncBlobStore();
- BlobBuilder blobBuilder =
- blobStore.blobBuilder( file.getName() ).payload( file ).calculateMD5().contentType( "text/plain" )
- .contentLength( file.length() );
-
- Blob blob = blobBuilder.build();
+ AWSCredentials credentials = new BasicAWSCredentials(accessId, secretKey);
+ ClientConfiguration clientConfig = new ClientConfiguration();
+ clientConfig.setProtocol( Protocol.HTTP);
- ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+ AmazonS3Client s3Client = new AmazonS3Client(credentials, clientConfig);
- LOG.info( "Uploaded file etag=" + futureETag.get() );
- }
- catch ( Exception e ) {
- LOG.error( "Error uploading to blob store", e );
- }
+ s3Client.createBucket( bucketName );
+ File uploadFile = new File( fileName );
+ PutObjectResult putObjectResult = s3Client.putObject( bucketName, uploadFile.getName(), uploadFile );
+ LOG.info("Uploaded file etag={}", putObjectResult.getETag());
}
@@ -453,7 +408,7 @@ public class WarehouseExport extends ExportingToolBase {
Query query = Query.fromQL( queryString );
query.setLimit( MAX_ENTITY_FETCH );
- query.setResultsLevel( Level.REFS );
+ query.setResultsLevel( Query.Level.REFS );
Results results = em.searchCollection( em.getApplicationRef(), collectionName, query );
while ( results.size() > 0 ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
index b0f4f6e..8402dd2 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/bean/MetricQuery.java
@@ -21,11 +21,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
-import org.apache.usergrid.persistence.AggregateCounter;
-import org.apache.usergrid.persistence.AggregateCounterSet;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.*;
import com.google.common.base.Preconditions;
import org.apache.usergrid.persistence.index.query.CounterResolution;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
index 446aa91..890c972 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportAppTest.java
@@ -19,25 +19,12 @@ package org.apache.usergrid.tools;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.ServiceITSuite;
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.management.OrganizationOwnerInfo;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityManager;
import org.junit.ClassRule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import rx.Scheduler;
-import rx.schedulers.Schedulers;
import java.io.File;
import java.io.FileFilter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -48,45 +35,45 @@ import static org.junit.Assert.assertTrue;
*/
public class ExportAppTest {
static final Logger logger = LoggerFactory.getLogger( ExportAppTest.class );
-
+
int NUM_COLLECTIONS = 10;
- int NUM_ENTITIES = 50;
+ int NUM_ENTITIES = 50;
int NUM_CONNECTIONS = 3;
@ClassRule
- public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
+ public static ServiceITSetup setup = new ServiceITSetupImpl();
@org.junit.Test
public void testBasicOperation() throws Exception {
-
+
String rand = RandomStringUtils.randomAlphanumeric( 10 );
-
+
// create app with some data
String orgName = "org_" + rand;
String appName = "app_" + rand;
-
+
ExportDataCreator creator = new ExportDataCreator();
creator.startTool( new String[] {
"-organization", orgName,
"-application", appName,
- "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort()
+ "-host", "localhost:9120"
}, false);
-
+
long start = System.currentTimeMillis();
-
+
String directoryName = "target/export" + rand;
ExportApp exportApp = new ExportApp();
exportApp.startTool( new String[]{
"-application", orgName + "/" + appName,
"-writeThreads", "100",
- "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
+ "-host", "localhost:9120",
"-outputDir", directoryName
}, false );
logger.info( "100 read and 100 write threads = " + (System.currentTimeMillis() - start) / 1000 + "s" );
-
+
File exportDir = new File(directoryName);
assertTrue( getFileCount( exportDir, "entities" ) > 0 );
assertTrue( getFileCount( exportDir, "connections" ) > 0 );
@@ -97,7 +84,7 @@ public class ExportAppTest {
exportApp.startTool( new String[]{
"-application", orgName + "/" + appName,
"-writeThreads", "1",
- "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
+ "-host", "localhost:9120",
"-outputDir", directoryName + "1"
}, false );
@@ -115,4 +102,4 @@ public class ExportAppTest {
}
} ).length;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b5144d05/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
index 9cce040..23ff096 100644
--- a/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/ExportImportAdminsTest.java
@@ -22,7 +22,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.ServiceITSuite;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.OrganizationOwnerInfo;
import org.apache.usergrid.management.UserInfo;
@@ -39,18 +39,20 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
-import java.util.*;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
import static junit.framework.TestCase.assertNotNull;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import static org.junit.Assert.*;
public class ExportImportAdminsTest {
static final Logger logger = LoggerFactory.getLogger( ExportImportAdminsTest.class );
-
+
@ClassRule
- public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
+ public static ServiceITSetup setup = new ServiceITSetupImpl();
@org.junit.Test
public void testExportUserAndOrg() throws Exception {
@@ -81,7 +83,7 @@ public class ExportImportAdminsTest {
ExportAdmins exportAdmins = new ExportAdmins();
exportAdmins.startTool( new String[] {
- "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
+ "-host", "localhost:9120",
"-outputDir", directoryName
}, false );
@@ -148,7 +150,7 @@ public class ExportImportAdminsTest {
public void testImportAdminUsersAndOrgs() throws Exception {
// first: generate the data file with unique user and org IDs and names
-
+
// data contains three users each with a unique org, one user has a duplicate email
String rand1 = RandomStringUtils.randomAlphanumeric( 10 );
@@ -214,7 +216,7 @@ public class ExportImportAdminsTest {
ImportAdmins importAdmins = new ImportAdmins();
importAdmins.startTool( new String[]{
- "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
+ "-host", "localhost:9120",
"-inputDir", tempDir.getAbsolutePath()
}, false );
@@ -229,7 +231,7 @@ public class ExportImportAdminsTest {
assertNotNull( "org 2 exists", orgInfo2 );
List<UserInfo> org2_users = setup.getMgmtSvc().getAdminUsersForOrganization( org_uuid_2 );
assertEquals( "org2 has two users", 2, org2_users.size() );
-
+
OrganizationInfo orgInfo3 = setup.getMgmtSvc().getOrganizationByUuid( org_uuid_3 );
assertNotNull( "org 3 exists", orgInfo3 );
List<UserInfo> org3_users = setup.getMgmtSvc().getAdminUsersForOrganization( org_uuid_3 );
@@ -237,7 +239,7 @@ public class ExportImportAdminsTest {
BiMap<UUID, String> user1_orgs = setup.getMgmtSvc().getOrganizationsForAdminUser( user_uuid_1 );
assertEquals( "user 1 has 2 orgs", 2, user1_orgs.size() );
-
+
BiMap<UUID, String> user2_orgs = setup.getMgmtSvc().getOrganizationsForAdminUser( user_uuid_2 );
assertEquals( "user 2 has two orgs gained one from duplicate", 2, user2_orgs.size() );
@@ -248,10 +250,10 @@ public class ExportImportAdminsTest {
logger.info("EXCEPTION EXPECTED");
}
- EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
+ EntityManager em = setup.getEmf().getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
Entity user3 = em.get( user_uuid_3 );
assertNull( "duplicate user does not exist", user3 );
}
-}
\ No newline at end of file
+}
[2/3] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-dev
Posted by sn...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-dev
Conflicts:
stack/core/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fedf1651
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fedf1651
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fedf1651
Branch: refs/heads/USERGRID-872
Commit: fedf16516b1f5eb26281b8d9cf6ed3b260c715f5
Parents: d32cee4 066d7db
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jul 22 09:46:21 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jul 22 09:46:21 2015 -0400
----------------------------------------------------------------------
stack/pom.xml | 2 +-
stack/tools/pom.xml | 6 +
.../org/apache/usergrid/tools/ExportAdmins.java | 117 ++--
.../org/apache/usergrid/tools/ExportApp.java | 536 +++++++++++++++++++
.../usergrid/tools/ExportDataCreator.java | 244 +++++++--
.../usergrid/tools/ExportingToolBase.java | 2 +-
.../org/apache/usergrid/tools/ImportAdmins.java | 226 ++++++--
stack/tools/src/main/resources/log4j.properties | 5 +
.../apache/usergrid/tools/ExportAppTest.java | 118 ++++
.../usergrid/tools/ExportImportAdminsTest.java | 71 ++-
...adata.usergrid-management.1433331614293.json | 52 ++
...users.usergrid-management.1433331614293.json | 12 +
12 files changed, 1224 insertions(+), 167 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fedf1651/stack/pom.xml
----------------------------------------------------------------------
diff --cc stack/pom.xml
index d7f279d,cc39e04..6f984e3
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@@ -109,10 -109,10 +109,10 @@@
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
- <org.springframework.version>3.1.2.RELEASE</org.springframework.version>
+ <org.springframework.version>3.2.13.RELEASE</org.springframework.version>
<shiro-version>1.2.3</shiro-version>
<slf4j-version>1.6.1</slf4j-version>
- <snakeyaml-version>1.8</snakeyaml-version>
+ <snakeyaml-version>1.9</snakeyaml-version>
<tomcat-version>7.0.59</tomcat-version>
<antlr.version>3.4</antlr.version>
<tika.version>1.4</tika.version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fedf1651/stack/tools/src/main/resources/log4j.properties
----------------------------------------------------------------------