You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/02/05 15:45:40 UTC
[11/16] incubator-usergrid git commit: Update by import test now
works.
Update by import test now works.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cdf63468
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cdf63468
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cdf63468
Branch: refs/heads/two-dot-o-import
Commit: cdf634685af004c2579212f8c7ea3866eba26045
Parents: 00e7194
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Feb 4 11:29:15 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Feb 4 11:29:15 2015 -0500
----------------------------------------------------------------------
.../management/importer/ImportServiceImpl.java | 9 +---
.../management/importer/ImportCollectionIT.java | 54 ++++++++------------
2 files changed, 23 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cdf63468/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index e122917..1ccc1db 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@ -797,7 +797,6 @@ public class ImportServiceImpl implements ImportService {
writeEvent.doWrite(em,fileImport);
}
};
-
// final AtomicLong entityCounter = new AtomicLong();
// final AtomicLong eventCounter = new AtomicLong();
@@ -805,7 +804,6 @@ public class ImportServiceImpl implements ImportService {
entityEventObservable.parallel(new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
@Override
public Observable<WriteEvent> call(Observable<WriteEvent> entityWrapperObservable) {
-
// TODO: need to fixed so that number of entities created can be counted correctly and
// TODO: also update last updated UUID for fileImport which is a must for resume-ability
@@ -840,14 +838,12 @@ public class ImportServiceImpl implements ImportService {
// }
// }
// );
-
return entityWrapperObservable.doOnNext(doWork);
-
}
}, Schedulers.io()).toBlocking().last();
jp.close();
- logger.debug("\n\nWrote entities\n");
+ logger.debug("\n\nimportEntitiesFromFile(): Wrote entities\n");
// now do other stuff: connections and dictionaries
entitiesOnly = false;
@@ -862,13 +858,12 @@ public class ImportServiceImpl implements ImportService {
@Override
public Observable<WriteEvent> call(Observable<WriteEvent> entityWrapperObservable) {
return entityWrapperObservable.doOnNext(doWork);
-
}
}, Schedulers.io()).toBlocking().last();
jp.close();
- logger.debug("\n\nWrote others\n");
+ logger.debug("\n\nimportEntitiesFromFile(): Wrote others\n");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cdf63468/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index f103602..0cc6bd4 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -18,24 +18,19 @@
package org.apache.usergrid.management.importer;
import com.amazonaws.SDKGlobalConfiguration;
-import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Service;
import com.google.inject.Module;
import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.batch.service.JobSchedulerService;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.management.export.ExportService;
-import org.apache.usergrid.management.export.S3Export;
-import org.apache.usergrid.management.export.S3ExportImpl;
import org.apache.usergrid.persistence.*;
-import org.apache.usergrid.persistence.entities.JobData;
import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.persistence.index.query.Query.Level;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
@@ -43,7 +38,6 @@ import org.apache.usergrid.services.notifications.QueueListener;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.domain.Blob;
import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.netty.config.NettyPayloadModule;
@@ -53,11 +47,8 @@ import org.slf4j.LoggerFactory;
import java.util.*;
-import static org.hamcrest.core.Is.is;
-import static org.hamcrest.core.IsNot.not;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
//@Concurrent
@@ -126,32 +117,27 @@ public class ImportCollectionIT {
@Test
public void testExportImportCollection() throws Exception {
- // clear bucket at start of test
- deleteBucket();
-
- final EntityManager emApp1 = setup.getEmf().getEntityManager( applicationId );
- // create a collection of "thing" entities in the first application
+ // create a collection of "thing" entities in the first application, export to S3
+ final EntityManager emApp1 = setup.getEmf().getEntityManager( applicationId );
Map<UUID, Entity> thingsMap = new HashMap<>();
List<Entity> things = new ArrayList<>();
-
createTestEntities(emApp1, thingsMap, things, "thing");
- // export the "things" collection to a JSON file in an S3 bucket
-
+ deleteBucket();
exportCollection( emApp1, "things" );
- // create new second application
- final UUID appId2 = setup.getMgmtSvc().createApplication(
- organization.getUuid(), "secondapp").getId();
+ // create new second application, import the data from S3
- // import the data into the new application
+ final UUID appId2 = setup.getMgmtSvc().createApplication(
+ organization.getUuid(), "second").getId();
final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
importCollection( emApp2, "things" );
+
// make sure that it worked
try {
@@ -238,13 +224,13 @@ public class ImportCollectionIT {
// create new second application and import those things from S3
final UUID appId2 = setup.getMgmtSvc().createApplication(
- organization.getUuid(), "secondapp").getId();
+ organization.getUuid(), "second").getId();
final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
importCollection( emApp2, "things" );
- // update the things in the new application, export to S3
+ // update the things in the second application, export to S3
for ( UUID uuid : thingsMap.keySet() ) {
Entity entity = emApp2.get( uuid );
@@ -253,10 +239,10 @@ public class ImportCollectionIT {
}
deleteBucket();
- exportCollection( emApp1, "things" );
+ exportCollection( emApp2, "things" );
- // import the updated things into the first application, check that they've been updated
+ // import the updated things back into the first application, check that they've been updated
importCollection( emApp1, "things" );
@@ -324,7 +310,7 @@ public class ImportCollectionIT {
*/
private void importCollection(final EntityManager em, final String collectionName ) throws Exception {
- logger.debug("\n\nImport into new app {}\n", em.getApplication().getUuid() );
+ logger.debug("\n\nImport into new app {}\n", em.getApplication().getName() );
ImportService importService = setup.getImportService();
UUID importUUID = importService.schedule( new HashMap<String, Object>() {{
@@ -360,13 +346,16 @@ public class ImportCollectionIT {
private void exportCollection(
final EntityManager em, final String collectionName ) throws Exception {
- logger.debug("\n\nExporting {} collection\n", collectionName );
+ logger.debug("\n\nExporting {} collection from application {}\n",
+ collectionName, em.getApplication().getName() );
+
+ em.refreshIndex();
ExportService exportService = setup.getExportService();
UUID exportUUID = exportService.schedule( new HashMap<String, Object>() {{
put( "path", organization.getName() + em.getApplication().getName());
put( "organizationId", organization.getUuid());
- put( "applicationId", applicationId );
+ put( "applicationId", em.getApplication().getUuid() );
put( "collectionName", collectionName);
put( "properties", new HashMap<String, Object>() {{
put( "storage_provider", "s3" );
@@ -393,7 +382,8 @@ public class ImportCollectionIT {
private void createTestEntities( final EntityManager em,
Map<UUID, Entity> thingsMap, List<Entity> things, final String type) throws Exception {
- logger.debug("\n\nCreating {} collection\n", type);
+ logger.debug("\n\nCreating new {} collection in application {}\n",
+ type, em.getApplication().getName() );
for ( int i = 0; i < 10; i++ ) {
final int count = i;
@@ -406,8 +396,6 @@ public class ImportCollectionIT {
things.add( e );
}
- logger.debug("\n\nCreate Connections\n");
-
// first two things are related to each other
em.createConnection( new SimpleEntityRef( "thing", things.get(0).getUuid()),
"related", new SimpleEntityRef( "thing", things.get(1).getUuid()));