You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/06/22 19:42:17 UTC
incubator-usergrid git commit: Added public method "getApplication"
because it was already being metered and public. Added fixes so the
MigrationTool would properly export all the entities and trimmed some of the
code from writing things we didn't need l
Repository: incubator-usergrid
Updated Branches:
refs/heads/MigrationTool 7f3111ae3 -> 6c6241082
Added public method "getApplication" because it was already being metered and public.
Added fixes so the MigrationTool would properly export all the entities and trimmed some of the code from writing things we didn't need like organizations.
Added a working test that verifies that all users are written to the file.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6c624108
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6c624108
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6c624108
Branch: refs/heads/MigrationTool
Commit: 6c6241082963fa5f620423d9ae1a20a1da988329
Parents: 7f3111a
Author: GERey <gr...@apigee.com>
Authored: Mon Jun 22 10:42:17 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Jun 22 10:42:17 2015 -0700
----------------------------------------------------------------------
.../persistence/EntityManagerFactory.java | 4 +
.../cassandra/EntityManagerFactoryImpl.java | 1 +
.../org/apache/usergrid/tools/Migration.java | 45 ++---
.../apache/usergrid/tools/MigrationTest.java | 168 +++++++++++++++++++
4 files changed, 198 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6c624108/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 1233a73..8b1d4a2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -20,6 +20,8 @@ package org.apache.usergrid.persistence;
import java.util.Map;
import java.util.UUID;
+import org.apache.usergrid.persistence.entities.Application;
+
// TODO: Auto-generated Javadoc
@@ -97,6 +99,8 @@ public interface EntityManagerFactory {
*/
public abstract Map<String, UUID> getApplications() throws Exception;
+ public Application getApplication( String name ) throws Exception;
+
public abstract void setup() throws Exception;
public abstract Map<String, String> getServiceProperties();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6c624108/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 53ccf40..fcfa392 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -300,6 +300,7 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
*
* @throws Exception the exception
*/
+ @Override
@Metered(group = "core", name = "EntityManagerFactory_getApplication")
public Application getApplication( String name ) throws Exception {
name = name.toLowerCase();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6c624108/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java b/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java
index 9bc27bf..b5c00c9 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/Migration.java
@@ -23,9 +23,12 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+
+import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.Results.Level;
import org.apache.usergrid.persistence.cassandra.CassandraService;
+import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.utils.JsonUtils;
import org.apache.usergrid.utils.StringUtils;
import org.codehaus.jackson.JsonGenerator;
@@ -78,6 +81,7 @@ public class Migration extends ExportingToolBase {
setVerbose( line );
+ applyOrgId( line );
//Checks for the application name that we want to export.
applyAppName( line );
if ( appName == null ){
@@ -97,7 +101,7 @@ public class Migration extends ExportingToolBase {
return;
}
} else {
- readThreadCount = 20;
+ readThreadCount = 1;
}
// start write queue worker
@@ -125,19 +129,13 @@ public class Migration extends ExportingToolBase {
Query query = new Query();
query.setLimit( MAX_ENTITY_FETCH );
query.setResultsLevel( Level.IDS );
- //EntityManager em = emf.getEntityManager( CassandraService.MANAGEMENT_APPLICATION_ID );
-
//Retrieve application that was given to the exporter
- UUID applicationUuid = emf.lookupApplication( "applicationName" );
+ OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( orgId );
+ Application exportedApplication = emf.getApplication( organizationInfo.getName()+"/"+appName );
+ UUID applicationUuid = exportedApplication.getUuid();
+ readQueue.add( applicationUuid );
EntityManager em = emf.getEntityManager( applicationUuid );
- //get all the collection names in an application
- Iterator<String> collections = em.getApplicationCollections().iterator();
-// while(collections.hasNext()){
-// String collectionName = collections.next();
-// managementService.get
-// em.getCollection
-// }
Map<String, Object> metadata = em.getApplicationCollectionMetadata();
echo( JsonUtils.mapToFormattedJsonString( metadata ) );
@@ -150,18 +148,21 @@ public class Migration extends ExportingToolBase {
query.setLimit( MAX_ENTITY_FETCH );
query.setResultsLevel( Level.IDS );
+ //loop until you get no cursor back.
Results ids = em.searchCollection( em.getApplicationRef(), collectionName, query );
- for (UUID uuid : ids.getIds()) {
- readQueue.add( uuid );
- logger.debug( "Added uuid to readQueue: " + uuid );
- }
- if (ids.getCursor() == null) {
- break;
- }
+ while(true) {
+ for ( UUID uuid : ids.getIds() ) {
+ readQueue.add( uuid );
+ logger.debug( "Added uuid to readQueue: " + uuid );
+ }
+ if ( ids.getCursor() == null ) {
+ break;
+ }
query.setCursor( ids.getCursor() );
ids = em.searchCollection( em.getApplicationRef(), collectionName, query );
+ }
}
adminUserWriter.setDone( true );
@@ -174,7 +175,8 @@ public class Migration extends ExportingToolBase {
}
public EntityManager applicationEntityManagerCreator() throws Exception{
- UUID applicationUuid = emf.lookupApplication( appName );
+ OrganizationInfo organizationInfo = managementService.getOrganizationByUuid( orgId );
+ UUID applicationUuid = emf.lookupApplication( organizationInfo.getName()+"/"+appName);
return emf.getEntityManager( applicationUuid );
}
@@ -187,7 +189,10 @@ public class Migration extends ExportingToolBase {
Option readThreads = OptionBuilder
.hasArg().withType(0).withDescription("Read Threads -" + READ_THREAD_COUNT).create(READ_THREAD_COUNT);
+ Option appName = OptionBuilder.hasArg().withType( 0 ).withDescription( "Application Name =").create( "appName" );
+
options.addOption( readThreads );
+ options.addOption( appName );
return options;
}
@@ -377,7 +382,7 @@ public class Migration extends ExportingToolBase {
echo( task.adminUser );
// write metadata to metadata file
- saveCollections( metadataFile, task );
+ //saveCollections( metadataFile, task );
saveConnections( metadataFile, task );
//saveOrganizations( metadataFile, task );
saveDictionaries( metadataFile, task );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6c624108/stack/tools/src/test/java/org/apache/usergrid/tools/MigrationTest.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/test/java/org/apache/usergrid/tools/MigrationTest.java b/stack/tools/src/test/java/org/apache/usergrid/tools/MigrationTest.java
new file mode 100644
index 0000000..16a099f
--- /dev/null
+++ b/stack/tools/src/test/java/org/apache/usergrid/tools/MigrationTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.tools;
+
+import com.google.common.collect.BiMap;
+import com.google.common.io.Files;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.usergrid.ServiceITSetup;
+import org.apache.usergrid.ServiceITSetupImpl;
+import org.apache.usergrid.ServiceITSuite;
+import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.OrganizationOwnerInfo;
+import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.utils.UUIDUtils;
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.junit.ClassRule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FilenameFilter;
+import java.util.*;
+
+import static junit.framework.TestCase.assertNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public class MigrationTest {
+ static final Logger logger = LoggerFactory.getLogger( MigrationTest.class );
+
+ @ClassRule
+ public static ServiceITSetup setup = new ServiceITSetupImpl( ServiceITSuite.cassandraResource );
+
+ @org.junit.Test
+ public void testExportUserAndOrg() throws Exception {
+
+
+ // Create orgOwner+Application
+ int numOfEntities = 10;
+
+ final String random1 = RandomStringUtils.randomAlphanumeric( 10 );
+
+ String applicationName = "app"+random1;
+
+
+ OrganizationOwnerInfo orgMade = null;
+ ApplicationInfo appMade = null;
+ //for ( int i = 0; i < 1; i++ ) {
+ orgMade = setup.getMgmtSvc().createOwnerAndOrganization(
+ "org_" + random1, "user_" + random1, "user_" + random1,
+ "user_" + random1 + "@example.com", "password" );
+ appMade = setup.getMgmtSvc().createApplication( orgMade.getOrganization().getUuid(), applicationName );
+
+ //for ( int j = 0; j < 5; j++ ) {
+
+ EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
+ customMaker.createApplicationCollection( "superappCol" + random1 );
+ //intialize user object to be posted
+ Map<String, Object> entityLevelProperties = null;
+ Entity[] entNotCopied;
+ entNotCopied = new Entity[numOfEntities];
+ //creates 10 entities
+ for ( int index = 0; index < numOfEntities; index++ ) {
+ entityLevelProperties = new LinkedHashMap<String, Object>();
+ entityLevelProperties.put( "username", "user_"+index );
+ entNotCopied[index] = customMaker.create( "superappCol" + random1, entityLevelProperties );
+ }
+ // }
+ // }
+
+
+ // export to file
+
+ String directoryName = "./target/export" + RandomStringUtils.randomAlphanumeric(10);
+
+ Migration migrationInstance = new Migration();
+ migrationInstance.startTool( new String[] {
+ "-host", "localhost:" + ServiceITSuite.cassandraResource.getRpcPort(),
+ "-orgId",orgMade.getOrganization().getUuid().toString(),"-appName",applicationName ,
+ "-outputDir", directoryName
+ }, false );
+
+ // read, parse and verify files
+
+ // first, the admin users file
+
+ File directory = new File( directoryName );
+ String[] adminUsersFileNames = directory.list( new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.startsWith("admin-users.");
+ }
+ });
+
+ // only one. read it into a map
+
+ File adminUsersFile = new File(
+ directory.getAbsolutePath() + File.separator + adminUsersFileNames[0] );
+
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode node = mapper.readTree( adminUsersFile );
+ assertTrue( node.isArray() );
+
+ // does file contain our two admin users?
+
+ Set<String> usernames = new HashSet<String>();
+ for ( int i=0; i<node.size(); i++) {
+ JsonNode jsonNode = node.get( i );
+ if ( jsonNode.get( "username" ) != null )
+ usernames.add( jsonNode.get("username").asText() );
+ }
+
+ //TODO: crappy way of doing verificatino that all users were written to the file.
+ for ( int i = 0; i < 10; i++ ) {
+ assertTrue( usernames.contains( "user_"+i ) );
+ }
+ //assertTrue( usernames.contains( "user_" + random2 ));
+
+ // second, the metadata file
+
+ String[] metadataFileNames = directory.list( new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.startsWith("admin-user-metadata.");
+ }
+ });
+
+ // only one, read it into a map
+
+ File metadataFile = new File(
+ directory.getAbsolutePath() + File.separator + metadataFileNames[0] );
+
+ mapper = new ObjectMapper();
+ node = mapper.readTree( metadataFile );
+ assertTrue( node.isObject() );
+
+ // do users belong to correct orgs
+
+// JsonNode user1node = node.findValue( orgOwnerInfo1.getOwner().getUuid().toString() );
+// JsonNode orgs1 = user1node.findValue( "organizations");
+// assertEquals( 2, orgs1.size() );
+//
+// JsonNode user2node = node.findValue( orgOwnerInfo2.getOwner().getUuid().toString() );
+// JsonNode orgs2 = user2node.findValue( "organizations");
+// assertEquals( 1, orgs2.size() );
+ }
+
+}
\ No newline at end of file