You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/07/24 17:22:05 UTC
[37/50] [abbrv] incubator-usergrid git commit: Merge branch 'master'
into two-dot-o
Merge branch 'master' into two-dot-o
Conflicts:
stack/core/pom.xml
stack/pom.xml
stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
stack/tools/src/main/resources/log4j.properties
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/066d7db4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/066d7db4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/066d7db4
Branch: refs/heads/USERGRID-869
Commit: 066d7db46786cc042cdc4fa28b56eb2f729561d1
Parents: 764a7c8 9b9f55a
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jul 22 09:44:58 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jul 22 09:44:58 2015 -0400
----------------------------------------------------------------------
stack/core/pom.xml | 1 -
stack/pom.xml | 2 +-
stack/tools/pom.xml | 6 +
.../org/apache/usergrid/tools/ExportAdmins.java | 117 ++--
.../org/apache/usergrid/tools/ExportApp.java | 536 +++++++++++++++++++
.../usergrid/tools/ExportDataCreator.java | 244 +++++++--
.../usergrid/tools/ExportingToolBase.java | 2 +-
.../org/apache/usergrid/tools/ImportAdmins.java | 226 ++++++--
.../org/apache/usergrid/tools/ToolBase.java | 2 +-
stack/tools/src/main/resources/log4j.properties | 5 +
.../apache/usergrid/tools/ExportAppTest.java | 118 ++++
.../usergrid/tools/ExportImportAdminsTest.java | 71 ++-
...adata.usergrid-management.1433331614293.json | 52 ++
...users.usergrid-management.1433331614293.json | 12 +
14 files changed, 1225 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/core/pom.xml
----------------------------------------------------------------------
diff --cc stack/core/pom.xml
index d91208d,f60dbc2..5e65ac3
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@@ -473,23 -573,16 +473,22 @@@
</dependency>
<dependency>
- <groupId>io.reactivex</groupId>
- <artifactId>rxjava</artifactId>
+ <groupId>com.netflix.rxjava</groupId>
+ <artifactId>rxjava-core</artifactId>
<version>${rx.version}</version>
</dependency>
-
-
<dependency>
- <groupId>io.reactivex</groupId>
+ <groupId>com.netflix.rxjava</groupId>
<artifactId>rxjava-math</artifactId>
- <version>1.0.0</version>
+ <version>${rx.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>com.clearspring.analytics</groupId>
+ <artifactId>stream</artifactId>
+ <version>2.7.0</version>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/pom.xml
----------------------------------------------------------------------
diff --cc stack/pom.xml
index cc61a7c,0e1b32d..cc39e04
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@@ -32,107 -32,84 +32,107 @@@
<description>Parent module for the Apache Usergrid Project</description>
<packaging>pom</packaging>
- <properties>
- <!-- =================================================================== -->
- <!-- Properties: Deployment Setting Defaults -->
- <!-- =================================================================== -->
- <!-- NOTE: override from the CLI or settings.xml -->
- <!-- NOTE: add server credentials config via settings -->
- <!-- NOTE: <settings> -->
- <!-- NOTE: <servers> -->
- <!-- NOTE: <server> -->
- <!-- NOTE: <id>usergrid.releases</id> -->
- <!-- NOTE: <username>akarasulu</username> -->
- <!-- NOTE: <password>*********</password> -->
- <!-- NOTE: </server> -->
- <!-- NOTE: <server> -->
- <!-- NOTE: <id>usergrid.snapshots</id> -->
- <!-- NOTE: <username>akarasulu</username> -->
- <!-- NOTE: <password>*********</password> -->
- <!-- NOTE: </server> -->
- <!-- NOTE: </servers> -->
- <!-- NOTE: -->
- <!-- NOTE: <profiles> -->
- <!-- NOTE: <profile> -->
- <!-- NOTE: <id>deployment</id> -->
- <!-- NOTE: <properties> -->
- <!-- NOTE: <release.repository.url> -->
- <!-- NOTE: https://to/your/custom/releases/repository -->
- <!-- NOTE: </release.repository.url> -->
- <!-- NOTE: <snapshot.repository.url> -->
- <!-- NOTE: https://to/your/custom/snapshots/repository -->
- <!-- NOTE: </shapshot.repository.url> -->
- <!-- NOTE: </properties> -->
- <!-- NOTE: </profile> -->
- <!-- NOTE: </profiles> -->
- <!-- NOTE: -->
- <!-- NOTE: <activeProfiles> -->
- <!-- NOTE: <activeProfile>deployment</activeProfile> -->
- <!-- NOTE: </activeProfiles> -->
- <!-- NOTE: </settings> -->
-
- <!-- =================================================================== -->
- <!-- Properties: General Settings -->
- <!-- =================================================================== -->
-
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-
- <!-- you can override these via MAVEN_OPTS -->
- <ug.heapmax>2048m</ug.heapmax>
- <ug.heapmin>2048m</ug.heapmin>
- <ug.argline>-Djava.awt.headless=true</ug.argline>
-
- <usergrid-custom-spring-properties>classpath:/usergrid-custom.properties</usergrid-custom-spring-properties>
- <usergrid-custom-spring-test-properties>classpath:/usergrid-custom-test.properties</usergrid-custom-spring-test-properties>
-
- <!-- =================================================================== -->
- <!-- Properties: Dependency Settings -->
- <!-- =================================================================== -->
-
- <amber-version>0.22-incubating</amber-version>
- <cassandra-version>1.2.12</cassandra-version>
- <hector-om-version>3.0-03</hector-om-version>
- <hector-version>1.1-4</hector-version>
- <hector-test-version>1.1-4</hector-test-version>
- <jackson-version>1.9.9</jackson-version>
- <jclouds.version>1.7.1</jclouds.version>
- <jersey-version>1.18</jersey-version>
- <junit-version>4.11</junit-version>
- <log4j-version>1.2.16</log4j-version>
- <metrics-version>2.1.2</metrics-version>
- <org.springframework.version>3.1.2.RELEASE</org.springframework.version>
- <shiro-version>1.2.0</shiro-version>
- <slf4j-version>1.6.1</slf4j-version>
- <snakeyaml-version>1.9</snakeyaml-version>
- <tomcat-version>7.0.42</tomcat-version>
- <antlr.version>3.4</antlr.version>
- <tika.version>1.4</tika.version>
- <metrics.version>3.0.0</metrics.version>
- <rx.version>1.0.12</rx.version>
- </properties>
+ <properties>
+ <!-- =================================================================== -->
+ <!-- Properties: Deployment Setting Defaults -->
+ <!-- =================================================================== -->
+ <!-- NOTE: override from the CLI or settings.xml -->
+ <!-- NOTE: add server credentials config via settings -->
+ <!-- NOTE: <settings> -->
+ <!-- NOTE: <servers> -->
+ <!-- NOTE: <server> -->
+ <!-- NOTE: <id>usergrid.releases</id> -->
+ <!-- NOTE: <username>akarasulu</username> -->
+ <!-- NOTE: <password>*********</password> -->
+ <!-- NOTE: </server> -->
+ <!-- NOTE: <server> -->
+ <!-- NOTE: <id>usergrid.snapshots</id> -->
+ <!-- NOTE: <username>akarasulu</username> -->
+ <!-- NOTE: <password>*********</password> -->
+ <!-- NOTE: </server> -->
+ <!-- NOTE: </servers> -->
+ <!-- NOTE: -->
+ <!-- NOTE: <profiles> -->
+ <!-- NOTE: <profile> -->
+ <!-- NOTE: <id>deployment</id> -->
+ <!-- NOTE: <properties> -->
+ <!-- NOTE: <release.repository.url> -->
+ <!-- NOTE: https://to/your/custom/releases/repository -->
+ <!-- NOTE: </release.repository.url> -->
+ <!-- NOTE: <snapshot.repository.url> -->
+ <!-- NOTE: https://to/your/custom/snapshots/repository -->
+ <!-- NOTE: </shapshot.repository.url> -->
+ <!-- NOTE: </properties> -->
+ <!-- NOTE: </profile> -->
+ <!-- NOTE: </profiles> -->
+ <!-- NOTE: -->
+ <!-- NOTE: <activeProfiles> -->
+ <!-- NOTE: <activeProfile>deployment</activeProfile> -->
+ <!-- NOTE: </activeProfiles> -->
+ <!-- NOTE: </settings> -->
+
+ <snapshot.repository.url>
+ https://repository.apache.org/content/repositories/snapshots
+ </snapshot.repository.url>
+ <release.repository.url>
+ https://repository.apache.org/service/local/staging/deploy/maven2
+ </release.repository.url>
+
+ <!-- =================================================================== -->
+ <!-- Properties: General Settings -->
+ <!-- =================================================================== -->
+
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+
+ <!-- you can override these via MAVEN_OPTS -->
+ <ug.heapmax>4096m</ug.heapmax>
+ <ug.heapmin>2048m</ug.heapmin>
+ <ug.argline>-Djava.awt.headless=true</ug.argline>
+
+ <usergrid-custom-spring-properties>classpath:/usergrid-deployment.properties</usergrid-custom-spring-properties>
+ <usergrid-custom-spring-test-properties>classpath:/usergrid-custom-test.properties</usergrid-custom-spring-test-properties>
+
+ <!-- =================================================================== -->
+ <!-- Properties: Dependency Settings -->
+ <!-- =================================================================== -->
+
+ <amber-version>0.22-incubating</amber-version>
+ <cassandra-version>1.2.18</cassandra-version>
+ <guava.version>18.0</guava.version>
+ <guice.version>4.0-beta5</guice.version>
+ <hector-om-version>3.0-03</hector-om-version>
+ <hector-version>1.1-4</hector-version>
+ <hector-test-version>1.1-4</hector-test-version>
+ <jackson-version>1.9.9</jackson-version>
+ <jackson-2-version>2.3.3</jackson-2-version>
+ <jclouds.version>1.8.0</jclouds.version>
+ <jersey-version>1.18.1</jersey-version>
+ <junit-version>4.12</junit-version>
+ <log4j-version>1.2.16</log4j-version>
+ <org.springframework.version>3.1.2.RELEASE</org.springframework.version>
+ <shiro-version>1.2.3</shiro-version>
+ <slf4j-version>1.6.1</slf4j-version>
- <snakeyaml-version>1.8</snakeyaml-version>
++ <snakeyaml-version>1.9</snakeyaml-version>
+ <tomcat-version>7.0.59</tomcat-version>
+ <antlr.version>3.4</antlr.version>
+ <tika.version>1.4</tika.version>
+ <mockito.version>1.10.8</mockito.version>
+
+ <!-- only use half the cores on the machine for testing -->
+ <usergrid.it.parallel>methods</usergrid.it.parallel>
+ <usergrid.it.reuseForks>true</usergrid.it.reuseForks>
+ <usergrid.it.forkCount>1</usergrid.it.forkCount>
+ <usergrid.it.threads>8</usergrid.it.threads>
+
+ <metrics.version>3.0.0</metrics.version>
+ <rx.version>0.19.6</rx.version>
+ <surefire.plugin.artifactName>surefire-junit47</surefire.plugin.artifactName>
+ <surefire.plugin.version>2.18.1</surefire.plugin.version>
+ <powermock.version>1.6.1</powermock.version>
+
+ <maven.build.timestamp.format>yyyy-MM-dd'T'HH-mm-ss'Z'</maven.build.timestamp.format>
+
+ </properties>
<licenses>
<license>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/tools/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index e175d01,d5dd42c..0bb74ab
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@@ -22,12 -23,13 +22,10 @@@ import org.apache.commons.cli.CommandLi
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
-import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
- import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
- import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.Results.Level;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.utils.StringUtils;
import org.codehaus.jackson.JsonGenerator;
import org.slf4j.Logger;
@@@ -57,17 -61,29 +55,29 @@@ import java.util.concurrent.atomic.Atom
* cassandra.lock.keyspace=My_Usergrid_Locks
*/
public class ExportAdmins extends ExportingToolBase {
-
+
static final Logger logger = LoggerFactory.getLogger( ExportAdmins.class );
-
++
public static final String ADMIN_USERS_PREFIX = "admin-users";
public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";
-
++
+ // map admin user UUID to list of organizations to which user belongs
+ private Map<UUID, List<Org>> userToOrgsMap = new HashMap<UUID, List<Org>>(50000);
+
+ private Map<String, UUID> orgNameToUUID = new HashMap<String, UUID>(50000);
-
++
+ private Set<UUID> orgsWritten = new HashSet<UUID>(50000);
-
++
+ private Set<UUID> duplicateOrgs = new HashSet<UUID>();
-
++
private static final String READ_THREAD_COUNT = "readThreads";
- private Map<String, List<Org>> orgMap = new HashMap<String, List<Org>>(80000);
private int readThreadCount;
- AtomicInteger count = new AtomicInteger( 0 );
+ AtomicInteger userCount = new AtomicInteger( 0 );
-
++
+ boolean ignoreInvalidUsers = false; // true to ignore users with no credentials or orgs
-
-
+
+
/**
* Represents an AdminUser that has been read and is ready for export.
*/
@@@ -169,10 -185,10 +179,10 @@@
while ( !done ) {
writeThread.join( 10000, 0 );
done = !writeThread.isAlive();
- logger.info( "Wrote {} users", count.get() );
+ logger.info( "Wrote {} users", userCount.get() );
}
}
-
+
@Override
@SuppressWarnings("static-access")
@@@ -207,10 -223,11 +217,11 @@@
organizations = em.searchCollection( em.getApplicationRef(), "groups", query );
for ( Entity organization : organizations.getEntities() ) {
execService.submit( new OrgMapWorker( organization ) );
+ count++;
}
- count++;
-
++
if ( count % 1000 == 0 ) {
- logger.info("Processed {} orgs for org map", count);
+ logger.info("Queued {} org map workers", count);
}
query.setCursor( organizations.getCursor() );
}
@@@ -218,8 -235,10 +229,10 @@@
execService.shutdown();
while ( !execService.awaitTermination( 10, TimeUnit.SECONDS ) ) {
- logger.info("Processed {} orgs for map", orgMap.size() );
+ logger.info( "Processed {} orgs for map", userToOrgsMap.size() );
}
-
++
+ logger.info("Org map complete, counted {} organizations", count);
}
@@@ -235,6 -254,7 +248,7 @@@
try {
final String orgName = orgEntity.getProperty( "path" ).toString();
final UUID orgId = orgEntity.getUuid();
-
++
for (UserInfo user : managementService.getAdminUsersForOrganization( orgEntity.getUuid() )) {
try {
Entity admin = managementService.getAdminUserEntityByUuid( user.getUuid() );
@@@ -297,10 -332,34 +326,34 @@@
AdminUserWriteTask task = new AdminUserWriteTask();
task.adminUser = entity;
- addDictionariesToTask( task, entity );
+ addDictionariesToTask( task, entity );
addOrganizationsToTask( task );
- writeQueue.add( task );
+ String actionTaken = "Processed";
+
+ if (ignoreInvalidUsers && (task.orgNamesByUuid.isEmpty()
+ || task.dictionariesByName.isEmpty()
+ || task.dictionariesByName.get( "credentials" ).isEmpty())) {
-
++
+ actionTaken = "Ignored";
-
++
+ } else {
+ writeQueue.add( task );
+ }
+
- Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty() ?
++ Map<String, Object> creds = (Map<String, Object>) (task.dictionariesByName.isEmpty() ?
+ 0 : task.dictionariesByName.get( "credentials" ));
-
++
+ logger.error( "{} admin user {}:{}:{} has organizations={} dictionaries={} credentials={}",
+ new Object[]{
+ actionTaken,
+ task.adminUser.getProperty( "username" ),
+ task.adminUser.getProperty( "email" ),
+ task.adminUser.getUuid(),
+ task.orgNamesByUuid.size(),
+ task.dictionariesByName.size(),
+ creds == null ? 0 : creds.size()
- } );
++ } );
} catch ( Exception e ) {
logger.error("Error reading data for user " + uuid, e );
@@@ -344,22 -391,17 +385,32 @@@
task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() );
++<<<<<<< HEAD
+ List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
+
++=======
+ List<Org> orgs = userToOrgsMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
-
++
++>>>>>>> master
if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {
-
++
+ // list of orgs from getOrganizationsForAdminUser() is less than expected, use userToOrgsMap
BiMap<UUID, String> bimap = HashBiMap.create();
for (Org org : orgs) {
bimap.put( org.orgId, org.orgName );
}
task.orgNamesByUuid = bimap;
}
++<<<<<<< HEAD
+
+ if ( task.orgNamesByUuid.isEmpty() ) {
+ logger.error("{}:{}:{} has no orgs", new Object[] {
+ task.adminUser.getProperty("username"),
+ task.adminUser.getProperty("email"),
+ task.adminUser.getUuid() } );
+ }
++=======
++>>>>>>> master
}
}
@@@ -485,6 -527,10 +536,10 @@@
jg.writeObject( orgs.get( uuid ) );
jg.writeEndObject();
-
++
+ synchronized (orgsWritten) {
+ orgsWritten.add( uuid );
+ }
}
jg.writeEndArray();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
index 7f8dd1b,3c427e1..c97dc9c
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ToolBase.java
@@@ -50,8 -50,8 +50,8 @@@ import static org.apache.usergrid.utils
/**
- * Base class for Usergrid Tools commands. Any class that implements this can be called with
- * Base class for Usergrid Tools commands. Any class that implements this can be called with java -jar {jarname}
- * org.apache.usergrid.tools.{classname}.
++ * Base class for Usergrid Tools commands. Any class that implements this can be called with
+ * java -jar {jarname} org.apache.usergrid.tools.{classname}.
*/
public abstract class ToolBase {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/066d7db4/stack/tools/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --cc stack/tools/src/main/resources/log4j.properties
index 00834cf,def47b4..18ebcc4
--- a/stack/tools/src/main/resources/log4j.properties
+++ b/stack/tools/src/main/resources/log4j.properties
@@@ -26,23 -26,8 +26,28 @@@ log4j.appender.stdout=org.apache.log4j.
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
++<<<<<<< HEAD
+log4j.category.org.apache.usergrid.tools=TRACE
+log4j.category.org.apache.usergrid=ERROR
+
+log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.DaoUtils=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.ConnectionRefImpl=WARN, stdout
+log4j.logger.me.prettyprint.cassandra.hector.TimingLogger=WARN, stdout
+log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
+log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
+#log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
+#log4j.logger.org.apache.usergrid.persistence.hector.CountingMutator=INFO, stdout
+
+log4j.logger.org.apache.usergrid.management.cassandra=DEBUB
+log4j.logger.org.apache.usergrid.tools=INFO
++=======
+ log4j.logger.org.apache.usergrid=INFO
+ log4j.logger.org.apache.usergrid.tools=DEBUG
++>>>>>>> master
log4j.logger.org.apache.usergrid.management.cassandra=WARN
log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN