You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/02/05 15:45:43 UTC
[14/16] incubator-usergrid git commit: Fixes to the
FileImportStatistics class.
Fixes to the FileImportStatistics class.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7e1c83df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7e1c83df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7e1c83df
Branch: refs/heads/two-dot-o-import
Commit: 7e1c83dfb7d4d07c9d24f362bad8d20af6804ce5
Parents: 1d0e1a1
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Feb 5 09:17:06 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Feb 5 09:17:06 2015 -0500
----------------------------------------------------------------------
.../persistence/entities/FileImport.java | 55 ++++++++++++++---
.../importer/FileImportStatistics.java | 62 +++++++-------------
.../management/importer/ImportServiceImpl.java | 58 +++++++-----------
.../management/export/ExportServiceIT.java | 7 +--
.../importer/FileImportStatisticsTest.java | 30 ++++++----
.../management/importer/ImportCollectionIT.java | 14 +++--
.../management/importer/ImportServiceIT.java | 6 +-
7 files changed, 122 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
index cd5ba2e..2f6ec87 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
@@ -21,6 +21,7 @@ import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.annotations.EntityProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import javax.xml.bind.annotation.XmlRootElement;
+import java.util.UUID;
/**
@@ -29,7 +30,7 @@ import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement
public class FileImport extends TypedEntity {
- //canceled , and expired states aren't used in current iteration.
+ //canceled , and expired states aren't used in current iteration.
public static enum State {
CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
}
@@ -44,11 +45,20 @@ public class FileImport extends TypedEntity {
protected String errorMessage;
/**
- * file name
+ * Name of file to import
*/
@EntityProperty
protected String fileName;
+ @EntityProperty
+ private UUID applicationId;
+
+ /**
+ * Target collection name
+ */
+ @EntityProperty
+ protected String collectionName;
+
/**
* File completion Status
*/
@@ -61,11 +71,6 @@ public class FileImport extends TypedEntity {
@EntityProperty
protected String lastUpdatedUUID;
-
-
- /**
- * File completion Status
- */
@EntityProperty
protected long importedEntityCount;
@@ -73,8 +78,6 @@ public class FileImport extends TypedEntity {
protected long failedEntityCount;
-
-
/**
* File completion Status
*/
@@ -86,6 +89,18 @@ public class FileImport extends TypedEntity {
public FileImport() {
+ setCompleted(false);
+ setLastUpdatedUUID(" ");
+ setErrorMessage(" ");
+ setState(FileImport.State.CREATED);
+ }
+
+
+ public FileImport( String fileName, UUID applicationId, String collectionName ) {
+ this();
+ this.fileName = fileName;
+ this.setApplicationId(applicationId);
+ this.collectionName = collectionName;
}
/**
@@ -172,6 +187,28 @@ public class FileImport extends TypedEntity {
this.fileName = fileName;
}
+ /**
+ * Get the collectionName of the target collection of the import.
+ * @return filename
+ */
+ public String getCollectionName() {
+ return collectionName;
+ }
+
+ public void setCollectionName( String collectionName ) {
+ this.collectionName = collectionName;
+ }
+
+ /**
+ * Target application name
+ */
+ public UUID getApplicationId() {
+ return applicationId;
+ }
+
+ public void setApplicationId(UUID applicationId) {
+ this.applicationId = applicationId;
+ }
public long getImportedEntityCount() {
return importedEntityCount;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
index 04ddcdf..8a4fa0e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
@@ -25,7 +25,9 @@ import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.entities.FailedImportConnection;
import org.apache.usergrid.persistence.entities.FailedImportEntity;
import org.apache.usergrid.persistence.entities.FileImport;
@@ -34,14 +36,14 @@ import org.apache.usergrid.persistence.exceptions.PersistenceException;
/**
- * Statistics used to track a file import. Only 1 instance of this class should exist per file imported in the cluster.
- * There is a direct 1-1 mapping of the statistics provided here and the file import status. This class is threadsafe to
- * be used across multiple threads.
+ * Statistics used to track a file import. Only 1 instance of this class should exist
+ * per file imported in the cluster. There is a direct 1-1 mapping of the statistics provided
+ * here and the file import status. This class is thread-safe to be used across multiple threads.
*/
public class FileImportStatistics {
- private static final String ERROR_MESSAGE = "Failed to import some data. See the import counters and errors.";
-
+ private static final String ERROR_MESSAGE =
+ "Failed to import some data. See the import counters and errors.";
private static final String ERRORS_CONNECTION_NAME = "errors";
@@ -54,23 +56,25 @@ public class FileImportStatistics {
private final Semaphore writeSemaphore = new Semaphore( 1 );
private final FileImport fileImport;
- private final EntityManager entityManager;
+ private final EntityManagerFactory emf;
private final int flushCount;
/**
- * Create an instance to track counters. Note that when this instance is created, it will attempt to load it's state
- * from the entity manager. In the case of using this when resuming, be sure you begin processing where the system thinks
- * it has left off.
+ * Create an instance to track counters. Note that when this instance is created, it will
+ * attempt to load it's state from the entity manager. In the case of using this when resuming,
+ * be sure you begin processing where the system thinks * it has left off.
*
- * @param entityManager The entity manager that will hold these entities.
- * @param fileImportId The uuid of the fileImport
+ * @param emf Entity Manager Factory
+ * @param fileImport File Import Entity
* @param flushCount The number of success + failures to accumulate before flushing
*/
- public FileImportStatistics( final EntityManager entityManager, final UUID fileImportId, final int flushCount ) {
- this.entityManager = entityManager;
+ public FileImportStatistics(
+ final EntityManagerFactory emf, final FileImport fileImport, final int flushCount ) {
+
+ this.emf = emf;
this.flushCount = flushCount;
- this.fileImport = getFileImport( fileImportId );
+ this.fileImport = fileImport;
this.entitiesWritten.addAndGet( fileImport.getImportedEntityCount() );
this.entitiesFailed.addAndGet( fileImport.getFailedEntityCount() );
@@ -89,8 +93,6 @@ public class FileImportStatistics {
}
-
-
/**
* Invoke when an entity fails to write correctly
*/
@@ -98,11 +100,11 @@ public class FileImportStatistics {
public void entityFailed( final String message ) {
entitiesFailed.incrementAndGet();
-
FailedImportEntity failedImportEntity = new FailedImportEntity();
failedImportEntity.setErrorMessage( message );
try {
+ EntityManager entityManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
failedImportEntity = entityManager.create( failedImportEntity );
entityManager.createConnection( fileImport, ERRORS_CONNECTION_NAME, failedImportEntity );
}
@@ -133,6 +135,7 @@ public class FileImportStatistics {
failedImportConnection.setErrorMessage( message );
try {
+ EntityManager entityManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
failedImportConnection = entityManager.create( failedImportConnection );
entityManager.createConnection( fileImport, ERRORS_CONNECTION_NAME, failedImportConnection );
}
@@ -306,34 +309,11 @@ public class FileImportStatistics {
fileImport.setState( state );
fileImport.setErrorMessage( message );
+ EntityManager entityManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
entityManager.update( fileImport );
}
catch ( Exception e ) {
throw new RuntimeException( "Unable to persist complete state", e );
}
}
-
-
- /**
- * Get the FileImport by uuid and return it
- *
- * @throws EntityNotFoundException if we can't find the file import with the given uuid
- */
- private FileImport getFileImport( final UUID fileImportId ) {
-
- final FileImport fileImport;
-
- try {
- fileImport = entityManager.get( fileImportId, FileImport.class );
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Unable to load fileImport with id " + fileImportId, e );
- }
-
- if ( fileImport == null ) {
- throw new EntityNotFoundException( "Could not file FileImport with id " + fileImportId );
- }
-
- return fileImport;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index 936e09b..1fea660 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@ -182,13 +182,9 @@ public class ImportServiceImpl implements ImportService {
}
// create a FileImport entity to store metadata about the fileImport job
- FileImport fileImport = new FileImport();
-
- fileImport.setFileName(file);
- fileImport.setCompleted(false);
- fileImport.setLastUpdatedUUID(" ");
- fileImport.setErrorMessage(" ");
- fileImport.setState(FileImport.State.CREATED);
+ String collectionName = config.get("collectionName").toString();
+ UUID applicationId = (UUID)config.get("applicationId");
+ FileImport fileImport = new FileImport( file, applicationId, collectionName );
fileImport = rootEm.create(fileImport);
Import importUG = rootEm.get(importRef, Import.class);
@@ -611,8 +607,7 @@ public class ImportServiceImpl implements ImportService {
File file = new File(queueMessage.getFileName());
UUID targetAppId = queueMessage.getApplicationId();
- // TODO: fix this or remove the dependency on ImportQueueMessage in this class
- parseFileToEntities( null, fileImport, file, targetAppId );
+ parseFileToEntities( fileImport, file, targetAppId );
}
@@ -624,15 +619,11 @@ public class ImportServiceImpl implements ImportService {
File file = new File(jobExecution.getJobData().getProperty("File").toString());
UUID targetAppId = (UUID) jobExecution.getJobData().getProperty("applicationId");
- Map<String, Object> config = (Map<String, Object>) jobExecution.getJobData().getProperty("importInfo");
- String collectionName = config.get("collectionName").toString();
-
- parseFileToEntities( collectionName, fileImport, file, targetAppId );
+ parseFileToEntities( fileImport, file, targetAppId );
}
- public void parseFileToEntities(
- String collectionName, FileImport fileImport, File file, UUID targetAppId ) throws Exception {
+ public void parseFileToEntities( FileImport fileImport, File file, UUID targetAppId ) throws Exception {
logger.debug("parseFileToEntities() for file {} ", file.getAbsolutePath());
@@ -657,7 +648,7 @@ public class ImportServiceImpl implements ImportService {
EntityManager targetEm = emf.getEntityManager(targetAppId);
logger.debug(" importing into app {} file {}", targetAppId.toString(), file.getAbsolutePath());
- importEntitiesFromFile( collectionName, file, targetEm, emManagementApp, fileImport );
+ importEntitiesFromFile( file, targetEm, emManagementApp, fileImport );
// Updates the state of file import job
@@ -751,7 +742,6 @@ public class ImportServiceImpl implements ImportService {
* @param fileImport The file import entity
*/
private void importEntitiesFromFile(
- final String collectionName,
final File file,
final EntityManager em,
final EntityManager rootEm,
@@ -764,23 +754,23 @@ public class ImportServiceImpl implements ImportService {
// observable that parses JSON and emits write events
JsonParser jp = getJsonParserForFile(file);
- //TODO, move the json parser into the observable creation so that open/close happens automatcially within the stream
+ // TODO: move the JSON parser into the observable creation
+ // so that open/close happens automatically within the stream
final JsonEntityParserObservable jsonObservableEntities =
- new JsonEntityParserObservable(jp, em, rootEm, collectionName, fileImport, entitiesOnly);
+ new JsonEntityParserObservable(jp, em, rootEm, fileImport, entitiesOnly);
final Observable<WriteEvent> entityEventObservable = Observable.create(jsonObservableEntities);
- //flush every 100 entities
- final FileImportStatistics statistics = new FileImportStatistics( em, fileImport.getUuid(), 100 );
- //truncate due to RX api
+ // flush every 100 entities
+ final FileImportStatistics statistics = new FileImportStatistics( emf, fileImport, 100 );
+
+ // truncate due to RX api
final int entityNumSkip = (int)statistics.getTotalEntityCount();
final int connectionNumSkip = (int)statistics.getTotalConnectionCount();
// function to execute for each write event
- /**
- * Function that invokes the work of the event.
- */
+ // function that invokes the work of the event.
final Action1<WriteEvent> doWork = new Action1<WriteEvent>() {
@Override
public void call( WriteEvent writeEvent ) {
@@ -788,12 +778,10 @@ public class ImportServiceImpl implements ImportService {
}
};
-
-
-
// start parsing JSON
- //only take while our stats tell us we should continue processing
- //potentially skip the first n if this is a resume operation
+
+ // only take while our stats tell us we should continue processing
+ // potentially skip the first n if this is a resume operation
entityEventObservable.takeWhile( new Func1<WriteEvent, Boolean>() {
@Override
public Boolean call( final WriteEvent writeEvent ) {
@@ -802,13 +790,10 @@ public class ImportServiceImpl implements ImportService {
} ).skip( entityNumSkip ).parallel( new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
@Override
public Observable<WriteEvent> call( Observable<WriteEvent> entityWrapperObservable ) {
-
-
return entityWrapperObservable.doOnNext( doWork );
}
}, Schedulers.io() ).toBlocking().last();
-
jp.close();
logger.debug("\n\nimportEntitiesFromFile(): Wrote entities\n");
@@ -820,7 +805,7 @@ public class ImportServiceImpl implements ImportService {
jp = getJsonParserForFile(file);
final JsonEntityParserObservable jsonObservableOther =
- new JsonEntityParserObservable(jp, em, rootEm, collectionName, fileImport, entitiesOnly);
+ new JsonEntityParserObservable(jp, em, rootEm, fileImport, entitiesOnly);
final Observable<WriteEvent> otherEventObservable = Observable.create(jsonObservableOther);
// only take while our stats tell us we should continue processing
@@ -973,7 +958,6 @@ public class ImportServiceImpl implements ImportService {
private final JsonParser jp;
EntityManager em;
EntityManager rootEm;
- String collectionName;
FileImport fileImport;
boolean entitiesOnly;
@@ -982,7 +966,6 @@ public class ImportServiceImpl implements ImportService {
JsonParser parser,
EntityManager em,
EntityManager rootEm,
- String collectionName,
FileImport fileImport,
boolean entitiesOnly) {
@@ -991,7 +974,6 @@ public class ImportServiceImpl implements ImportService {
this.rootEm = rootEm;
this.fileImport = fileImport;
this.entitiesOnly = entitiesOnly;
- this.collectionName = collectionName;
}
@@ -1007,7 +989,7 @@ public class ImportServiceImpl implements ImportService {
boolean done = false;
// we ignore imported entity type information, entities get the type of the collection
- String collectionType = InflectionUtils.singularize( collectionName );
+ String collectionType = InflectionUtils.singularize( fileImport.getCollectionName() );
Stack tokenStack = new Stack();
EntityRef lastEntity = null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
index a7fa0cd..d23d358 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.UUID;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
@@ -98,7 +99,8 @@ public class ExportServiceIT {
private OrganizationInfo organization;
private UUID applicationId;
-
+ final String bucketName = System.getProperty( "bucketName" )
+ + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
@Before
public void setup() throws Exception {
@@ -805,7 +807,6 @@ public class ExportServiceIT {
;
}
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
@@ -897,7 +898,6 @@ public class ExportServiceIT {
Thread.sleep( 3000 );
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
@@ -1007,7 +1007,6 @@ public class ExportServiceIT {
;
}
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java
index 510302a..d2ce2c0 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java
@@ -23,6 +23,8 @@ package org.apache.usergrid.management.importer;
import java.util.List;
import java.util.UUID;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.EntityManagerFactory;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.invocation.InvocationOnMock;
@@ -52,18 +54,16 @@ public class FileImportStatisticsTest {
@Test
public void testSuccess() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
-
final FileImport fileImport = new FileImport();
fileImport.setUuid( importFileId );
- when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
-
-
- final FileImportStatistics fileImportStatistics = new FileImportStatistics( em, importFileId, 1000 );
+ final FileImportStatistics fileImportStatistics = new FileImportStatistics( emf, fileImport, 1000 );
final long expectedCount = 100;
@@ -93,15 +93,15 @@ public class FileImportStatisticsTest {
@Test
public void testBoth() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
final FileImport fileImport = new FileImport();
- fileImport.setUuid( importFileId );
-
- when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
+ fileImport.setUuid(importFileId);
//mock up returning the FailedEntityImport instance after save is invoked.
@@ -112,7 +112,7 @@ public class FileImportStatisticsTest {
}
} );
- final FileImportStatistics fileImportStatistics = new FileImportStatistics( em, importFileId, 1000 );
+ final FileImportStatistics fileImportStatistics = new FileImportStatistics( emf, fileImport, 1000 );
final long expectedSuccess = 100;
@@ -171,7 +171,9 @@ public class FileImportStatisticsTest {
@Test
public void explicitFail() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
@@ -182,7 +184,7 @@ public class FileImportStatisticsTest {
when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
- final FileImportStatistics fileImportStatistics = new FileImportStatistics( em, importFileId, 1000 );
+ final FileImportStatistics fileImportStatistics = new FileImportStatistics( emf, fileImport, 1000 );
final long expectedCount = 100;
@@ -216,7 +218,9 @@ public class FileImportStatisticsTest {
@Test
public void testAutoFlushSuccess() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
@@ -246,7 +250,7 @@ public class FileImportStatisticsTest {
/ expectedFlushCount;
//set this to 1/2, so that we get saved twice
- final FileImportStatistics fileImportStatistics = new FileImportStatistics( em, importFileId, flushSize );
+ final FileImportStatistics fileImportStatistics = new FileImportStatistics( emf, fileImport, flushSize );
for ( long i = 0; i < expectedSuccess; i++ ) {
@@ -326,7 +330,9 @@ public class FileImportStatisticsTest {
@Test
public void loadingExistingState() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
@@ -342,7 +348,7 @@ public class FileImportStatisticsTest {
//mock up returning the FailedEntityImport instance after save is invoked.
- FileImportStatistics statistics = new FileImportStatistics( em, importFileId, 100 );
+ FileImportStatistics statistics = new FileImportStatistics( emf, fileImport, 100 );
assertEquals( 1, statistics.getEntitiesWritten() );
assertEquals( 2, statistics.getEntitiesFailed() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index 6b080cc..168cec7 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -52,7 +52,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-//@Concurrent
+//@Concurrent These tests cannot be run concurrently
public class ImportCollectionIT {
private static final Logger logger = LoggerFactory.getLogger(ImportCollectionIT.class);
@@ -67,6 +67,8 @@ public class ImportCollectionIT {
QueueListener listener;
+ final String bucketName = System.getProperty( "bucketName" )
+ + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
@Rule
public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
@@ -268,6 +270,8 @@ public class ImportCollectionIT {
@Test
public void testImportWithWrongTypes() throws Exception {
+ deleteBucket();
+
// create an app with a collection of cats, export it to S3
String appName = "import-test-" + RandomStringUtils.randomAlphanumeric(10);
@@ -302,6 +306,8 @@ public class ImportCollectionIT {
@Test
public void testImportWithMultipleFiles() throws Exception {
+ deleteBucket();
+
// create 10 applications each with collection of 10 things, export all to S3
Map<UUID, Entity> thingsMap = new HashMap<>();
@@ -353,6 +359,7 @@ public class ImportCollectionIT {
logger.debug("\n\nImport into new app {}\n", em.getApplication().getName() );
+
ImportService importService = setup.getImportService();
UUID importUUID = importService.schedule( new HashMap<String, Object>() {{
put( "path", organization.getName() + em.getApplication().getName());
@@ -366,7 +373,7 @@ public class ImportCollectionIT {
System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ) );
- put( "bucket_location", System.getProperty( "bucketName" ) );
+ put( "bucket_location", bucketName);
}});
}});
}});
@@ -406,7 +413,7 @@ public class ImportCollectionIT {
System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ) );
- put( "bucket_location", System.getProperty( "bucketName" ) );
+ put( "bucket_location", bucketName );
}});
}});
}});
@@ -460,7 +467,6 @@ public class ImportCollectionIT {
logger.debug("\n\nDelete bucket\n");
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
index adf3a4d..f93c68b 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
@@ -22,6 +22,7 @@ import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Service;
import com.google.inject.Module;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
@@ -77,6 +78,8 @@ public class ImportServiceIT {
QueueListener listener;
+ final String bucketName = System.getProperty( "bucketName" )
+ + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
@Rule
public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
@@ -569,7 +572,7 @@ public class ImportServiceIT {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> properties = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
- storage_info.put( "bucket_location", System.getProperty( "bucketName" ) );
+ storage_info.put( "bucket_location", bucketName );
storage_info.put( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
@@ -611,7 +614,6 @@ public class ImportServiceIT {
// delete the s3 bucket which was created for testing
public void deleteBucket() {
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );