You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/06/29 20:01:38 UTC
[8/8] incubator-usergrid git commit: Merge branch 'master' into
two-dot-o-dev
Merge branch 'master' into two-dot-o-dev
Conflicts:
.usergridversion
portal/css/main.min.css
stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
stack/tools/src/main/resources/log4j.properties
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8195cb09
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8195cb09
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8195cb09
Branch: refs/heads/two-dot-o-dev
Commit: 8195cb090c553c62fc38bc87f131d65ef9ffc207
Parents: 5731a01 655392e
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Jun 29 14:00:43 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Jun 29 14:00:43 2015 -0400
----------------------------------------------------------------------
.usergridversion | 1 +
LICENSE | 41 +--
NOTICE | 3 +-
.../css/arsmarquette/ARSMaquettePro-Light.otf | Bin 184600 -> 0 bytes
.../css/arsmarquette/ARSMaquettePro-Medium.otf | Bin 188020 -> 0 bytes
.../css/arsmarquette/ARSMaquettePro-Regular.otf | Bin 188096 -> 0 bytes
portal/css/main.css | 15 --
portal/css/main.min.css | 2 +-
release/release-candidate.sh | 10 -
.../org/apache/usergrid/tools/ExportAdmins.java | 268 ++++++++++---------
.../org/apache/usergrid/tools/ImportAdmins.java | 252 ++++++++---------
stack/tools/src/main/resources/log4j.properties | 5 +-
12 files changed, 293 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8195cb09/.usergridversion
----------------------------------------------------------------------
diff --cc .usergridversion
index e69de29,276823b..6753471
--- a/.usergridversion
+++ b/.usergridversion
@@@ -1,0 -1,1 +1,1 @@@
-1.0.2-SNAPSHOT
++2.0.0-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8195cb09/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
----------------------------------------------------------------------
diff --cc stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
index 75064a2,d3d6371..5ab5b9d
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/ExportAdmins.java
@@@ -14,18 -14,19 +14,17 @@@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.usergrid.tools;
-
import com.google.common.collect.BiMap;
+ import com.google.common.collect.HashBiMap;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
- import org.apache.usergrid.corepersistence.util.CpNamingUtils;
- import org.apache.usergrid.persistence.ConnectionRef;
+ import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.Results.Level;
import org.apache.usergrid.persistence.cassandra.CassandraService;
@@@ -35,19 -35,30 +34,35 @@@ import org.codehaus.jackson.JsonGenerat
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+ import java.util.concurrent.atomic.AtomicInteger;
+
+ import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
/**
- * Export Admin Users and metadata including organizations.
+ * Export Admin Users and metadata including organizations and passwords.
+ *
+ * Usage Example:
- *
+ *
- * java -jar usergrid-tools.jar ExportAdmins
+ * java -Xmx8000m -Dlog4j.configuration=file:/home/me/log4j.properties -classpath . \
+ * -jar usergrid-tools-1.0.2.jar ImportAdmins -writeThreads 100 -auditThreads 100 \
+ * -host casshost -inputDir=/home/me/export-data
- *
++ *
+ * If you want to provide any property overrides, put properties file named usergrid-custom-tools.properties
+ * in the same directory where you run the above command. For example, you might want to set the Cassandra
+ * client threads and export from a specific set of keyspaces:
- *
++ *
+ * cassandra.connections=110
+ * cassandra.system.keyspace=My_Usergrid
+ * cassandra.application.keyspace=My_Usergrid_Applications
+ * cassandra.lock.keyspace=My_Usergrid_Locks
*/
public class ExportAdmins extends ExportingToolBase {
@@@ -55,9 -66,12 +70,12 @@@
public static final String ADMIN_USERS_PREFIX = "admin-users";
public static final String ADMIN_USER_METADATA_PREFIX = "admin-user-metadata";
private static final String READ_THREAD_COUNT = "readThreads";
+ private Map<String, List<Org>> orgMap = new HashMap<String, List<Org>>(80000);
private int readThreadCount;
-
+
+ AtomicInteger count = new AtomicInteger( 0 );
-
-
++
+
/**
* Represents an AdminUser that has been read and is ready for export.
*/
@@@ -69,8 -81,21 +85,21 @@@
BiMap<UUID, String> orgNamesByUuid;
}
-
+
/**
+ * Represents an organization associated with a user.
+ */
+ private class Org {
+ UUID orgId;
+ String orgName;
+ public Org( UUID orgId, String orgName ) {
+ this.orgId = orgId;
+ this.orgName = orgName;
+ }
+ }
+
+
+ /**
* Export admin users using multiple threads.
* <p/>
* How it works:
@@@ -100,6 -125,8 +129,8 @@@
readThreadCount = 20;
}
+ buildOrgMap();
-
++
// start write queue worker
BlockingQueue<AdminUserWriteTask> writeQueue = new LinkedBlockingQueue<AdminUserWriteTask>();
@@@ -146,9 -173,15 +177,15 @@@
}
logger.debug( "Waiting for write thread to complete" );
- writeThread.join();
-
++
+ boolean done = false;
+ while ( !done ) {
+ writeThread.join( 10000, 0 );
+ done = !writeThread.isAlive();
+ logger.info( "Wrote {} users", count.get() );
+ }
}
-
+
@Override
@SuppressWarnings("static-access")
@@@ -164,6 -197,76 +201,76 @@@
}
+ /**
+ * Shouldn't have to do this but getOrganizationsForAdminUser() is not 100% reliable in some Usergrid installations.
+ */
+ private void buildOrgMap() throws Exception {
+
+ logger.info("Building org map");
+
+ ExecutorService execService = Executors.newFixedThreadPool( this.readThreadCount );
+
+ EntityManager em = emf.getEntityManager( MANAGEMENT_APPLICATION_ID );
+ String queryString = "select *";
+ Query query = Query.fromQL( queryString );
+ query.withLimit( 1000 );
+ Results organizations = null;
+ int count = 0;
+ do {
+ organizations = em.searchCollection( em.getApplicationRef(), "groups", query );
+ for ( Entity organization : organizations.getEntities() ) {
+ execService.submit( new OrgMapWorker( organization ) );
+ }
+ count++;
+ if ( count % 1000 == 0 ) {
+ logger.info("Processed {} orgs for org map", count);
+ }
+ query.setCursor( organizations.getCursor() );
+ }
+ while ( organizations != null && organizations.hasCursor() );
+
+ execService.shutdown();
+ while ( !execService.awaitTermination( 10, TimeUnit.SECONDS ) ) {
+ logger.info("Processed {} orgs for map", orgMap.size() );
+ }
+ }
+
+
+ public class OrgMapWorker implements Runnable {
+ private final Entity orgEntity;
-
++
+ public OrgMapWorker( Entity orgEntity ) {
+ this.orgEntity = orgEntity;
+ }
-
++
+ @Override
+ public void run() {
+ try {
+ final String orgName = orgEntity.getProperty( "path" ).toString();
+ final UUID orgId = orgEntity.getUuid();
+ for (UserInfo user : managementService.getAdminUsersForOrganization( orgEntity.getUuid() )) {
+ try {
+ Entity admin = managementService.getAdminUserEntityByUuid( user.getUuid() );
+ List<Org> orgs = orgMap.get( admin.getProperty( "username" ) );
+ if (orgs == null) {
+ orgs = new ArrayList<Org>();
+ orgMap.put( admin.getProperty( "username" ).toString().toLowerCase(), orgs );
+ }
+ orgs.add( new Org( orgId, orgName ) );
+
+ //logger.debug("Added org {} for user {}", orgName, admin.getProperty( "username" ));
+
+ } catch (Exception e) {
+ logger.warn( "Cannot get orgs for userId {}", user.getUuid() );
+ }
+ }
+ } catch ( Exception e ) {
+ logger.error("Error getting users for org {}:{}", orgEntity.getName(), orgEntity.getUuid());
+ }
+ }
+ }
+
+
public class AdminUserReader implements Runnable {
private boolean done = true;
@@@ -256,7 -326,11 +330,11 @@@
EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
Set<String> dictionaries = em.getDictionaries( entity );
-
+
+ if ( dictionaries.isEmpty() ) {
+ logger.error("User {}:{} has no dictionaries", task.adminUser.getName(), task.adminUser.getUuid() );
+ }
+
task.dictionariesByName = new HashMap<String, Map<Object, Object>>();
for (String dictionary : dictionaries) {
@@@ -265,32 -339,26 +343,26 @@@
}
}
+ private void addOrganizationsToTask(AdminUserWriteTask task) throws Exception {
- private void addConnectionsToTask(AdminUserWriteTask task, Entity entity) throws Exception {
- EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
-
- task.connectionsByType = new HashMap<String, List<ConnectionRef>>();
-
- Set<String> connectionTypes = em.getConnectionTypes( entity );
- for (String connectionType : connectionTypes) {
+ task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( task.adminUser.getUuid() );
- List<ConnectionRef> connRefs = task.connectionsByType.get( connectionType );
- if ( connRefs == null ) {
- connRefs = new ArrayList<ConnectionRef>();
- }
-
- Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Level.IDS );
- List<ConnectionRef> connections = results.getConnections();
+ List<Org> orgs = orgMap.get( task.adminUser.getProperty( "username" ).toString().toLowerCase() );
-
+
- for (ConnectionRef connectionRef : connections) {
- connRefs.add( connectionRef );
+ if ( orgs != null && task.orgNamesByUuid.size() < orgs.size() ) {
+ BiMap<UUID, String> bimap = HashBiMap.create();
+ for (Org org : orgs) {
+ bimap.put( org.orgId, org.orgName );
}
+ task.orgNamesByUuid = bimap;
}
- }
-
+
-
- private void addOrganizationsToTask(AdminUserWriteTask task, Entity entity) throws Exception {
- task.orgNamesByUuid = managementService.getOrganizationsForAdminUser( entity.getUuid() );
+ if ( task.orgNamesByUuid.isEmpty() ) {
+ logger.error("{}:{}:{} has no orgs", new Object[] {
- task.adminUser.getProperty("username"),
- task.adminUser.getProperty("email"),
++ task.adminUser.getProperty("username"),
++ task.adminUser.getProperty("email"),
+ task.adminUser.getUuid() } );
+ }
}
public void setDone(boolean done) {
@@@ -347,18 -413,20 +417,20 @@@
echo( task.adminUser );
// write metadata to metadata file
- saveCollections( metadataFile, task );
- saveConnections( metadataFile, task );
+ metadataFile.writeFieldName( task.adminUser.getUuid().toString() );
+ metadataFile.writeStartObject();
+
saveOrganizations( metadataFile, task );
- saveDictionaries( metadataFile, task );
+ saveDictionaries( metadataFile, task );
-
+
- logger.debug("Exported user {}", task.adminUser.getProperty( "email" ));
+ metadataFile.writeEndObject();
-
+
- count++;
- if ( count % 1000 == 0 ) {
- logger.info("Exported {} admin users", count);
- }
+ logger.debug( "Exported user {}:{}:{}", new Object[] {
+ task.adminUser.getProperty("username"),
+ task.adminUser.getProperty("email"),
+ task.adminUser.getUuid() } );
+ count.addAndGet( 1 );
} catch (InterruptedException e) {
throw new Exception("Interrupted", e);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8195cb09/stack/tools/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --cc stack/tools/src/main/resources/log4j.properties
index c288fec,6cf0a92..037f4b1
--- a/stack/tools/src/main/resources/log4j.properties
+++ b/stack/tools/src/main/resources/log4j.properties
@@@ -26,25 -26,9 +26,25 @@@ log4j.appender.stdout=org.apache.log4j.
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d %p (%t) [%c] - %m%n
- <<<<<<< HEAD
+log4j.category.org.apache.usergrid.tools=TRACE
+log4j.category.org.apache.usergrid=ERROR
+
+log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.DaoUtils=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerImpl=WARN, stdout
+log4j.logger.org.apache.usergrid.persistence.cassandra.ConnectionRefImpl=WARN, stdout
+log4j.logger.me.prettyprint.cassandra.hector.TimingLogger=WARN, stdout
+log4j.logger.org.apache.usergrid.rest.security.AllowAjaxFilter=WARN, stdout
+log4j.logger.me.prettyprint.hector.api.beans.AbstractComposite=ERROR, stdout
+#log4j.logger.org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl=DEBUG, stdout
+#log4j.logger.org.apache.usergrid.persistence.hector.CountingMutator=INFO, stdout
- =======
+log4j.logger.org.apache.usergrid.tools=DEBUG
+log4j.logger.org.apache.usergrid.management.cassandra=DEBUB
+ log4j.logger.org.apache.usergrid.tools=INFO
+ log4j.logger.org.apache.usergrid.management.cassandra=WARN
log4j.logger.org.apache.usergrid.persistence.cassandra.DB=WARN
log4j.logger.org.apache.usergrid.persistence.cassandra.BATCH=WARN
log4j.logger.org.apache.usergrid.persistence.cassandra.EntityManagerFactoryImpl=WARN
@@@ -58,15 -42,5 +58,14 @@@ log4j.logger.me.prettyprint.hector.api.
log4j.logger.org.apache.usergrid.persistence.hector.CountingMutator=INFO
- >>>>>>> master
#log4j.logger.org.apache.cassandra.service.StorageProxy=DEBUG, stdout
+#log4j.logger.org.apache.usergrid.corepersistence=INFO
+#log4j.logger.org.apache.usergrid.corepersistence.CpSetup=INFO
+log4j.logger.org.apache.usergrid.corepersistence.CpEntityManagerFactory=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.CpEntityManager=DEBUG
+#log4j.logger.org.apache.usergrid.corepersistence.CpRelationManager=DEBUG
+
+#log4j.logger.org.apache.usergrid.persistence.collection=INFO
+#log4j.logger.org.apache.usergrid.persistence.index=DEBUG
+#log4j.logger.org.apache.usergrid.persistence.index.impl=DEBUG