You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/02/05 15:45:30 UTC
[01/16] incubator-usergrid git commit: compile issue...
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-import 57ae8164b -> fd4d47bb3
compile issue...
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0fc961ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0fc961ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0fc961ed
Branch: refs/heads/two-dot-o-import
Commit: 0fc961ed80238d6408693e79b79a31a2d0d04bb2
Parents: f675224
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Jan 27 09:10:16 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Jan 27 09:10:16 2015 -0700
----------------------------------------------------------------------
.../usergrid/services/notifications/ProviderAdapterFactory.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0fc961ed/stack/services/src/main/java/org/apache/usergrid/services/notifications/ProviderAdapterFactory.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ProviderAdapterFactory.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ProviderAdapterFactory.java
index 9a48c35..2ee9042 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ProviderAdapterFactory.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ProviderAdapterFactory.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.services.notifications;
-import com.sun.javafx.tools.ant.Application;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.entities.Notifier;
import org.apache.usergrid.services.notifications.apns.APNsAdapter;
[14/16] incubator-usergrid git commit: Fixes to the
FileImportStatistics class.
Posted by sn...@apache.org.
Fixes to the FileImportStatistics class.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7e1c83df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7e1c83df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7e1c83df
Branch: refs/heads/two-dot-o-import
Commit: 7e1c83dfb7d4d07c9d24f362bad8d20af6804ce5
Parents: 1d0e1a1
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Feb 5 09:17:06 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Feb 5 09:17:06 2015 -0500
----------------------------------------------------------------------
.../persistence/entities/FileImport.java | 55 ++++++++++++++---
.../importer/FileImportStatistics.java | 62 +++++++-------------
.../management/importer/ImportServiceImpl.java | 58 +++++++-----------
.../management/export/ExportServiceIT.java | 7 +--
.../importer/FileImportStatisticsTest.java | 30 ++++++----
.../management/importer/ImportCollectionIT.java | 14 +++--
.../management/importer/ImportServiceIT.java | 6 +-
7 files changed, 122 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
index cd5ba2e..2f6ec87 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/FileImport.java
@@ -21,6 +21,7 @@ import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.annotations.EntityProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import javax.xml.bind.annotation.XmlRootElement;
+import java.util.UUID;
/**
@@ -29,7 +30,7 @@ import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement
public class FileImport extends TypedEntity {
- //canceled , and expired states aren't used in current iteration.
+ //canceled , and expired states aren't used in current iteration.
public static enum State {
CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
}
@@ -44,11 +45,20 @@ public class FileImport extends TypedEntity {
protected String errorMessage;
/**
- * file name
+ * Name of file to import
*/
@EntityProperty
protected String fileName;
+ @EntityProperty
+ private UUID applicationId;
+
+ /**
+ * Target collection name
+ */
+ @EntityProperty
+ protected String collectionName;
+
/**
* File completion Status
*/
@@ -61,11 +71,6 @@ public class FileImport extends TypedEntity {
@EntityProperty
protected String lastUpdatedUUID;
-
-
- /**
- * File completion Status
- */
@EntityProperty
protected long importedEntityCount;
@@ -73,8 +78,6 @@ public class FileImport extends TypedEntity {
protected long failedEntityCount;
-
-
/**
* File completion Status
*/
@@ -86,6 +89,18 @@ public class FileImport extends TypedEntity {
public FileImport() {
+ setCompleted(false);
+ setLastUpdatedUUID(" ");
+ setErrorMessage(" ");
+ setState(FileImport.State.CREATED);
+ }
+
+
+ public FileImport( String fileName, UUID applicationId, String collectionName ) {
+ this();
+ this.fileName = fileName;
+ this.setApplicationId(applicationId);
+ this.collectionName = collectionName;
}
/**
@@ -172,6 +187,28 @@ public class FileImport extends TypedEntity {
this.fileName = fileName;
}
+ /**
+ * Get the collectionName of the target collection of the import.
+ * @return filename
+ */
+ public String getCollectionName() {
+ return collectionName;
+ }
+
+ public void setCollectionName( String collectionName ) {
+ this.collectionName = collectionName;
+ }
+
+ /**
+ * Target application name
+ */
+ public UUID getApplicationId() {
+ return applicationId;
+ }
+
+ public void setApplicationId(UUID applicationId) {
+ this.applicationId = applicationId;
+ }
public long getImportedEntityCount() {
return importedEntityCount;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
index 04ddcdf..8a4fa0e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportStatistics.java
@@ -25,7 +25,9 @@ import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
import org.apache.usergrid.persistence.entities.FailedImportConnection;
import org.apache.usergrid.persistence.entities.FailedImportEntity;
import org.apache.usergrid.persistence.entities.FileImport;
@@ -34,14 +36,14 @@ import org.apache.usergrid.persistence.exceptions.PersistenceException;
/**
- * Statistics used to track a file import. Only 1 instance of this class should exist per file imported in the cluster.
- * There is a direct 1-1 mapping of the statistics provided here and the file import status. This class is threadsafe to
- * be used across multiple threads.
+ * Statistics used to track a file import. Only 1 instance of this class should exist
+ * per file imported in the cluster. There is a direct 1-1 mapping of the statistics provided
+ * here and the file import status. This class is thread-safe to be used across multiple threads.
*/
public class FileImportStatistics {
- private static final String ERROR_MESSAGE = "Failed to import some data. See the import counters and errors.";
-
+ private static final String ERROR_MESSAGE =
+ "Failed to import some data. See the import counters and errors.";
private static final String ERRORS_CONNECTION_NAME = "errors";
@@ -54,23 +56,25 @@ public class FileImportStatistics {
private final Semaphore writeSemaphore = new Semaphore( 1 );
private final FileImport fileImport;
- private final EntityManager entityManager;
+ private final EntityManagerFactory emf;
private final int flushCount;
/**
- * Create an instance to track counters. Note that when this instance is created, it will attempt to load it's state
- * from the entity manager. In the case of using this when resuming, be sure you begin processing where the system thinks
- * it has left off.
+ * Create an instance to track counters. Note that when this instance is created, it will
+ * attempt to load it's state from the entity manager. In the case of using this when resuming,
+ * be sure you begin processing where the system thinks * it has left off.
*
- * @param entityManager The entity manager that will hold these entities.
- * @param fileImportId The uuid of the fileImport
+ * @param emf Entity Manager Factory
+ * @param fileImport File Import Entity
* @param flushCount The number of success + failures to accumulate before flushing
*/
- public FileImportStatistics( final EntityManager entityManager, final UUID fileImportId, final int flushCount ) {
- this.entityManager = entityManager;
+ public FileImportStatistics(
+ final EntityManagerFactory emf, final FileImport fileImport, final int flushCount ) {
+
+ this.emf = emf;
this.flushCount = flushCount;
- this.fileImport = getFileImport( fileImportId );
+ this.fileImport = fileImport;
this.entitiesWritten.addAndGet( fileImport.getImportedEntityCount() );
this.entitiesFailed.addAndGet( fileImport.getFailedEntityCount() );
@@ -89,8 +93,6 @@ public class FileImportStatistics {
}
-
-
/**
* Invoke when an entity fails to write correctly
*/
@@ -98,11 +100,11 @@ public class FileImportStatistics {
public void entityFailed( final String message ) {
entitiesFailed.incrementAndGet();
-
FailedImportEntity failedImportEntity = new FailedImportEntity();
failedImportEntity.setErrorMessage( message );
try {
+ EntityManager entityManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
failedImportEntity = entityManager.create( failedImportEntity );
entityManager.createConnection( fileImport, ERRORS_CONNECTION_NAME, failedImportEntity );
}
@@ -133,6 +135,7 @@ public class FileImportStatistics {
failedImportConnection.setErrorMessage( message );
try {
+ EntityManager entityManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
failedImportConnection = entityManager.create( failedImportConnection );
entityManager.createConnection( fileImport, ERRORS_CONNECTION_NAME, failedImportConnection );
}
@@ -306,34 +309,11 @@ public class FileImportStatistics {
fileImport.setState( state );
fileImport.setErrorMessage( message );
+ EntityManager entityManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
entityManager.update( fileImport );
}
catch ( Exception e ) {
throw new RuntimeException( "Unable to persist complete state", e );
}
}
-
-
- /**
- * Get the FileImport by uuid and return it
- *
- * @throws EntityNotFoundException if we can't find the file import with the given uuid
- */
- private FileImport getFileImport( final UUID fileImportId ) {
-
- final FileImport fileImport;
-
- try {
- fileImport = entityManager.get( fileImportId, FileImport.class );
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Unable to load fileImport with id " + fileImportId, e );
- }
-
- if ( fileImport == null ) {
- throw new EntityNotFoundException( "Could not file FileImport with id " + fileImportId );
- }
-
- return fileImport;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index 936e09b..1fea660 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@ -182,13 +182,9 @@ public class ImportServiceImpl implements ImportService {
}
// create a FileImport entity to store metadata about the fileImport job
- FileImport fileImport = new FileImport();
-
- fileImport.setFileName(file);
- fileImport.setCompleted(false);
- fileImport.setLastUpdatedUUID(" ");
- fileImport.setErrorMessage(" ");
- fileImport.setState(FileImport.State.CREATED);
+ String collectionName = config.get("collectionName").toString();
+ UUID applicationId = (UUID)config.get("applicationId");
+ FileImport fileImport = new FileImport( file, applicationId, collectionName );
fileImport = rootEm.create(fileImport);
Import importUG = rootEm.get(importRef, Import.class);
@@ -611,8 +607,7 @@ public class ImportServiceImpl implements ImportService {
File file = new File(queueMessage.getFileName());
UUID targetAppId = queueMessage.getApplicationId();
- // TODO: fix this or remove the dependency on ImportQueueMessage in this class
- parseFileToEntities( null, fileImport, file, targetAppId );
+ parseFileToEntities( fileImport, file, targetAppId );
}
@@ -624,15 +619,11 @@ public class ImportServiceImpl implements ImportService {
File file = new File(jobExecution.getJobData().getProperty("File").toString());
UUID targetAppId = (UUID) jobExecution.getJobData().getProperty("applicationId");
- Map<String, Object> config = (Map<String, Object>) jobExecution.getJobData().getProperty("importInfo");
- String collectionName = config.get("collectionName").toString();
-
- parseFileToEntities( collectionName, fileImport, file, targetAppId );
+ parseFileToEntities( fileImport, file, targetAppId );
}
- public void parseFileToEntities(
- String collectionName, FileImport fileImport, File file, UUID targetAppId ) throws Exception {
+ public void parseFileToEntities( FileImport fileImport, File file, UUID targetAppId ) throws Exception {
logger.debug("parseFileToEntities() for file {} ", file.getAbsolutePath());
@@ -657,7 +648,7 @@ public class ImportServiceImpl implements ImportService {
EntityManager targetEm = emf.getEntityManager(targetAppId);
logger.debug(" importing into app {} file {}", targetAppId.toString(), file.getAbsolutePath());
- importEntitiesFromFile( collectionName, file, targetEm, emManagementApp, fileImport );
+ importEntitiesFromFile( file, targetEm, emManagementApp, fileImport );
// Updates the state of file import job
@@ -751,7 +742,6 @@ public class ImportServiceImpl implements ImportService {
* @param fileImport The file import entity
*/
private void importEntitiesFromFile(
- final String collectionName,
final File file,
final EntityManager em,
final EntityManager rootEm,
@@ -764,23 +754,23 @@ public class ImportServiceImpl implements ImportService {
// observable that parses JSON and emits write events
JsonParser jp = getJsonParserForFile(file);
- //TODO, move the json parser into the observable creation so that open/close happens automatcially within the stream
+ // TODO: move the JSON parser into the observable creation
+ // so that open/close happens automatically within the stream
final JsonEntityParserObservable jsonObservableEntities =
- new JsonEntityParserObservable(jp, em, rootEm, collectionName, fileImport, entitiesOnly);
+ new JsonEntityParserObservable(jp, em, rootEm, fileImport, entitiesOnly);
final Observable<WriteEvent> entityEventObservable = Observable.create(jsonObservableEntities);
- //flush every 100 entities
- final FileImportStatistics statistics = new FileImportStatistics( em, fileImport.getUuid(), 100 );
- //truncate due to RX api
+ // flush every 100 entities
+ final FileImportStatistics statistics = new FileImportStatistics( emf, fileImport, 100 );
+
+ // truncate due to RX api
final int entityNumSkip = (int)statistics.getTotalEntityCount();
final int connectionNumSkip = (int)statistics.getTotalConnectionCount();
// function to execute for each write event
- /**
- * Function that invokes the work of the event.
- */
+ // function that invokes the work of the event.
final Action1<WriteEvent> doWork = new Action1<WriteEvent>() {
@Override
public void call( WriteEvent writeEvent ) {
@@ -788,12 +778,10 @@ public class ImportServiceImpl implements ImportService {
}
};
-
-
-
// start parsing JSON
- //only take while our stats tell us we should continue processing
- //potentially skip the first n if this is a resume operation
+
+ // only take while our stats tell us we should continue processing
+ // potentially skip the first n if this is a resume operation
entityEventObservable.takeWhile( new Func1<WriteEvent, Boolean>() {
@Override
public Boolean call( final WriteEvent writeEvent ) {
@@ -802,13 +790,10 @@ public class ImportServiceImpl implements ImportService {
} ).skip( entityNumSkip ).parallel( new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
@Override
public Observable<WriteEvent> call( Observable<WriteEvent> entityWrapperObservable ) {
-
-
return entityWrapperObservable.doOnNext( doWork );
}
}, Schedulers.io() ).toBlocking().last();
-
jp.close();
logger.debug("\n\nimportEntitiesFromFile(): Wrote entities\n");
@@ -820,7 +805,7 @@ public class ImportServiceImpl implements ImportService {
jp = getJsonParserForFile(file);
final JsonEntityParserObservable jsonObservableOther =
- new JsonEntityParserObservable(jp, em, rootEm, collectionName, fileImport, entitiesOnly);
+ new JsonEntityParserObservable(jp, em, rootEm, fileImport, entitiesOnly);
final Observable<WriteEvent> otherEventObservable = Observable.create(jsonObservableOther);
// only take while our stats tell us we should continue processing
@@ -973,7 +958,6 @@ public class ImportServiceImpl implements ImportService {
private final JsonParser jp;
EntityManager em;
EntityManager rootEm;
- String collectionName;
FileImport fileImport;
boolean entitiesOnly;
@@ -982,7 +966,6 @@ public class ImportServiceImpl implements ImportService {
JsonParser parser,
EntityManager em,
EntityManager rootEm,
- String collectionName,
FileImport fileImport,
boolean entitiesOnly) {
@@ -991,7 +974,6 @@ public class ImportServiceImpl implements ImportService {
this.rootEm = rootEm;
this.fileImport = fileImport;
this.entitiesOnly = entitiesOnly;
- this.collectionName = collectionName;
}
@@ -1007,7 +989,7 @@ public class ImportServiceImpl implements ImportService {
boolean done = false;
// we ignore imported entity type information, entities get the type of the collection
- String collectionType = InflectionUtils.singularize( collectionName );
+ String collectionType = InflectionUtils.singularize( fileImport.getCollectionName() );
Stack tokenStack = new Stack();
EntityRef lastEntity = null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
index a7fa0cd..d23d358 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.UUID;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
@@ -98,7 +99,8 @@ public class ExportServiceIT {
private OrganizationInfo organization;
private UUID applicationId;
-
+ final String bucketName = System.getProperty( "bucketName" )
+ + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
@Before
public void setup() throws Exception {
@@ -805,7 +807,6 @@ public class ExportServiceIT {
;
}
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
@@ -897,7 +898,6 @@ public class ExportServiceIT {
Thread.sleep( 3000 );
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
@@ -1007,7 +1007,6 @@ public class ExportServiceIT {
;
}
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java
index 510302a..d2ce2c0 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportStatisticsTest.java
@@ -23,6 +23,8 @@ package org.apache.usergrid.management.importer;
import java.util.List;
import java.util.UUID;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.EntityManagerFactory;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.invocation.InvocationOnMock;
@@ -52,18 +54,16 @@ public class FileImportStatisticsTest {
@Test
public void testSuccess() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
-
final FileImport fileImport = new FileImport();
fileImport.setUuid( importFileId );
- when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
-
-
- final FileImportStatistics fileImportStatistics = new FileImportStatistics( em, importFileId, 1000 );
+ final FileImportStatistics fileImportStatistics = new FileImportStatistics( emf, fileImport, 1000 );
final long expectedCount = 100;
@@ -93,15 +93,15 @@ public class FileImportStatisticsTest {
@Test
public void testBoth() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
final FileImport fileImport = new FileImport();
- fileImport.setUuid( importFileId );
-
- when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
+ fileImport.setUuid(importFileId);
//mock up returning the FailedEntityImport instance after save is invoked.
@@ -112,7 +112,7 @@ public class FileImportStatisticsTest {
}
} );
- final FileImportStatistics fileImportStatistics = new FileImportStatistics( em, importFileId, 1000 );
+ final FileImportStatistics fileImportStatistics = new FileImportStatistics( emf, fileImport, 1000 );
final long expectedSuccess = 100;
@@ -171,7 +171,9 @@ public class FileImportStatisticsTest {
@Test
public void explicitFail() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
@@ -182,7 +184,7 @@ public class FileImportStatisticsTest {
when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
- final FileImportStatistics fileImportStatistics = new FileImportStatistics( em, importFileId, 1000 );
+ final FileImportStatistics fileImportStatistics = new FileImportStatistics( emf, fileImport, 1000 );
final long expectedCount = 100;
@@ -216,7 +218,9 @@ public class FileImportStatisticsTest {
@Test
public void testAutoFlushSuccess() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
@@ -246,7 +250,7 @@ public class FileImportStatisticsTest {
/ expectedFlushCount;
//set this to 1/2, so that we get saved twice
- final FileImportStatistics fileImportStatistics = new FileImportStatistics( em, importFileId, flushSize );
+ final FileImportStatistics fileImportStatistics = new FileImportStatistics( emf, fileImport, flushSize );
for ( long i = 0; i < expectedSuccess; i++ ) {
@@ -326,7 +330,9 @@ public class FileImportStatisticsTest {
@Test
public void loadingExistingState() throws Exception {
+ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
final EntityManager em = mock( EntityManager.class );
+ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
final UUID importFileId = UUIDGenerator.newTimeUUID();
@@ -342,7 +348,7 @@ public class FileImportStatisticsTest {
//mock up returning the FailedEntityImport instance after save is invoked.
- FileImportStatistics statistics = new FileImportStatistics( em, importFileId, 100 );
+ FileImportStatistics statistics = new FileImportStatistics( emf, fileImport, 100 );
assertEquals( 1, statistics.getEntitiesWritten() );
assertEquals( 2, statistics.getEntitiesFailed() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index 6b080cc..168cec7 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -52,7 +52,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-//@Concurrent
+//@Concurrent These tests cannot be run concurrently
public class ImportCollectionIT {
private static final Logger logger = LoggerFactory.getLogger(ImportCollectionIT.class);
@@ -67,6 +67,8 @@ public class ImportCollectionIT {
QueueListener listener;
+ final String bucketName = System.getProperty( "bucketName" )
+ + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
@Rule
public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
@@ -268,6 +270,8 @@ public class ImportCollectionIT {
@Test
public void testImportWithWrongTypes() throws Exception {
+ deleteBucket();
+
// create an app with a collection of cats, export it to S3
String appName = "import-test-" + RandomStringUtils.randomAlphanumeric(10);
@@ -302,6 +306,8 @@ public class ImportCollectionIT {
@Test
public void testImportWithMultipleFiles() throws Exception {
+ deleteBucket();
+
// create 10 applications each with collection of 10 things, export all to S3
Map<UUID, Entity> thingsMap = new HashMap<>();
@@ -353,6 +359,7 @@ public class ImportCollectionIT {
logger.debug("\n\nImport into new app {}\n", em.getApplication().getName() );
+
ImportService importService = setup.getImportService();
UUID importUUID = importService.schedule( new HashMap<String, Object>() {{
put( "path", organization.getName() + em.getApplication().getName());
@@ -366,7 +373,7 @@ public class ImportCollectionIT {
System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ) );
- put( "bucket_location", System.getProperty( "bucketName" ) );
+ put( "bucket_location", bucketName);
}});
}});
}});
@@ -406,7 +413,7 @@ public class ImportCollectionIT {
System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ) );
- put( "bucket_location", System.getProperty( "bucketName" ) );
+ put( "bucket_location", bucketName );
}});
}});
}});
@@ -460,7 +467,6 @@ public class ImportCollectionIT {
logger.debug("\n\nDelete bucket\n");
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7e1c83df/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
index adf3a4d..f93c68b 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportServiceIT.java
@@ -22,6 +22,7 @@ import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Service;
import com.google.inject.Module;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
@@ -77,6 +78,8 @@ public class ImportServiceIT {
QueueListener listener;
+ final String bucketName = System.getProperty( "bucketName" )
+ + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
@Rule
public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
@@ -569,7 +572,7 @@ public class ImportServiceIT {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> properties = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
- storage_info.put( "bucket_location", System.getProperty( "bucketName" ) );
+ storage_info.put( "bucket_location", bucketName );
storage_info.put( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
@@ -611,7 +614,6 @@ public class ImportServiceIT {
// delete the s3 bucket which was created for testing
public void deleteBucket() {
- String bucketName = System.getProperty( "bucketName" );
String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
[10/16] incubator-usergrid git commit: Merge branch
'two-dot-o-import' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-import
Posted by sn...@apache.org.
Merge branch 'two-dot-o-import' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-import
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/00e71943
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/00e71943
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/00e71943
Branch: refs/heads/two-dot-o-import
Commit: 00e71943588264b985f125e0635327f4afe68693
Parents: 3cdf50d c9742c0
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Feb 4 10:28:27 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Feb 4 10:28:27 2015 -0500
----------------------------------------------------------------------
.../persistence/entities/FileImport.java | 31 ++++
.../importer/FileImportStatistics.java | 153 +++++++++++++++++++
.../importer/FileImportStatisticsTest.java | 133 ++++++++++++++++
3 files changed, 317 insertions(+)
----------------------------------------------------------------------
[12/16] incubator-usergrid git commit: Implementing new import tests.
Posted by sn...@apache.org.
Implementing new import tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b47904e6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b47904e6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b47904e6
Branch: refs/heads/two-dot-o-import
Commit: b47904e66b21ad2d3df59399edf0c00b02b405ab
Parents: cdf6346
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Feb 4 14:52:45 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Feb 4 14:52:45 2015 -0500
----------------------------------------------------------------------
.../management/importer/ImportServiceImpl.java | 36 +++---
.../management/importer/S3ImportImpl.java | 5 +-
.../management/importer/ImportCollectionIT.java | 114 +++++++++++++------
3 files changed, 103 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b47904e6/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index 1ccc1db..2054804 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.entities.FileImport;
import org.apache.usergrid.persistence.entities.Import;
import org.apache.usergrid.persistence.entities.JobData;
+import org.apache.usergrid.utils.InflectionUtils;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonParseException;
import org.codehaus.jackson.JsonParser;
@@ -584,7 +585,7 @@ public class ImportServiceImpl implements ImportService {
* @param type it indicates the type of import
*/
public ArrayList<File> copyFileFromS3(Import importUG, String appFileName,
- Map<String, Object> config, S3Import s3Import, ImportType type) throws Exception {
+ Map<String, Object> config, S3Import s3Import, ImportType type) throws Exception {
EntityManager rootEm = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
ArrayList<File> copyFiles = new ArrayList<>();
@@ -610,7 +611,8 @@ public class ImportServiceImpl implements ImportService {
File file = new File(queueMessage.getFileName());
UUID targetAppId = queueMessage.getApplicationId();
- parseFileToEntities( fileImport, file, targetAppId );
+ // TODO: fix this or remove the dependency on ImportQueueMessage in this class
+ parseFileToEntities( null, fileImport, file, targetAppId );
}
@@ -622,11 +624,15 @@ public class ImportServiceImpl implements ImportService {
File file = new File(jobExecution.getJobData().getProperty("File").toString());
UUID targetAppId = (UUID) jobExecution.getJobData().getProperty("applicationId");
- parseFileToEntities( fileImport, file, targetAppId );
+ Map<String, Object> config = (Map<String, Object>) jobExecution.getJobData().getProperty("importInfo");
+ String collectionName = config.get("collectionName").toString();
+
+ parseFileToEntities( collectionName, fileImport, file, targetAppId );
}
- public void parseFileToEntities( FileImport fileImport, File file, UUID targetAppId ) throws Exception {
+ public void parseFileToEntities(
+ String collectionName, FileImport fileImport, File file, UUID targetAppId ) throws Exception {
logger.debug("parseFileToEntities() for file {} ", file.getAbsolutePath());
@@ -651,7 +657,7 @@ public class ImportServiceImpl implements ImportService {
EntityManager targetEm = emf.getEntityManager(targetAppId);
logger.debug(" importing into app {} file {}", targetAppId.toString(), file.getAbsolutePath());
- importEntitiesFromFile(file, targetEm, emManagementApp, fileImport );
+ importEntitiesFromFile( collectionName, file, targetEm, emManagementApp, fileImport );
// TODO: fix the resume on error feature
@@ -774,6 +780,7 @@ public class ImportServiceImpl implements ImportService {
* @param fileImport The file import entity
*/
private void importEntitiesFromFile(
+ final String collectionName,
final File file,
final EntityManager em,
final EntityManager rootEm,
@@ -786,7 +793,7 @@ public class ImportServiceImpl implements ImportService {
// observable that parses JSON and emits write events
JsonParser jp = getJsonParserForFile(file);
final JsonEntityParserObservable jsonObservableEntities =
- new JsonEntityParserObservable(jp, em, rootEm, fileImport, entitiesOnly);
+ new JsonEntityParserObservable(jp, em, rootEm, collectionName, fileImport, entitiesOnly);
final Observable<WriteEvent> entityEventObservable = Observable.create(jsonObservableEntities);
// function to execute for each write event
@@ -851,7 +858,7 @@ public class ImportServiceImpl implements ImportService {
// observable that parses JSON and emits write events
jp = getJsonParserForFile(file);
final JsonEntityParserObservable jsonObservableOther =
- new JsonEntityParserObservable(jp, em, rootEm, fileImport, entitiesOnly);
+ new JsonEntityParserObservable(jp, em, rootEm, collectionName, fileImport, entitiesOnly);
final Observable<WriteEvent> otherEventObservable = Observable.create(jsonObservableOther);
otherEventObservable.parallel(new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
@@ -1011,6 +1018,7 @@ public class ImportServiceImpl implements ImportService {
private final JsonParser jp;
EntityManager em;
EntityManager rootEm;
+ String collectionName;
FileImport fileImport;
boolean entitiesOnly;
@@ -1019,13 +1027,16 @@ public class ImportServiceImpl implements ImportService {
JsonParser parser,
EntityManager em,
EntityManager rootEm,
+ String collectionName,
FileImport fileImport,
boolean entitiesOnly) {
+
this.jp = parser;
this.em = em;
this.rootEm = rootEm;
this.fileImport = fileImport;
this.entitiesOnly = entitiesOnly;
+ this.collectionName = collectionName;
}
@@ -1038,12 +1049,12 @@ public class ImportServiceImpl implements ImportService {
private void process(final Subscriber<? super WriteEvent> subscriber) {
try {
-
boolean done = false;
- // TODO: replace stack with counter or some other mechanism
- Stack tokenStack = new Stack();
+ // we ignore imported entity type information, entities get the type of the collection
+ String collectionType = InflectionUtils.singularize( collectionName );
+ Stack tokenStack = new Stack();
EntityRef lastEntity = null;
while (!done) {
@@ -1062,13 +1073,12 @@ public class ImportServiceImpl implements ImportService {
Map<String, Object> entityMap = jp.readValueAs(HashMap.class);
- String type = (String) entityMap.get("type");
UUID uuid = UUID.fromString((String) entityMap.get("uuid"));
- lastEntity = new SimpleEntityRef(type, uuid);
+ lastEntity = new SimpleEntityRef(collectionType, uuid);
logger.debug("{}Got entity with uuid {}", indent, lastEntity);
if (entitiesOnly) {
- WriteEvent event = new EntityEvent(uuid, type, entityMap);
+ WriteEvent event = new EntityEvent(uuid, collectionType, entityMap);
subscriber.onNext(event);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b47904e6/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
index 4a71da1..0fefeb0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
@@ -114,8 +114,9 @@ public class S3ImportImpl implements S3Import {
if (pfn.collectionName == null) {
errors.add("Filename does not specify collection name");
- } else if (!pfn.collectionName.equals(importInfo.get("collectionName"))) {
- errors.add("Collection name in input file should be " + pfn.collectionName);
+ // we shouldn't care what collection name is specified in the import file.
+// } else if (!pfn.collectionName.equals(importInfo.get("collectionName"))) {
+// errors.add("Collection name in input file should be " + pfn.collectionName);
}
if (!errors.isEmpty()) {
throw new IllegalArgumentException("Input errors " + errors.toString());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b47904e6/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index 0cc6bd4..6b080cc 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -21,6 +21,7 @@ import com.amazonaws.SDKGlobalConfiguration;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Service;
import com.google.inject.Module;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
@@ -111,13 +112,22 @@ public class ImportCollectionIT {
}
Assume.assumeTrue( configured );
- }
+ }
+
+
+ @After
+ public void after() throws Exception {
+ if(listener != null) {
+ listener.stop();
+ listener = null;
+ }
+ }
+
// test case to check if a collection file is imported correctly
@Test
public void testExportImportCollection() throws Exception {
-
// create a collection of "thing" entities in the first application, export to S3
final EntityManager emApp1 = setup.getEmf().getEntityManager( applicationId );
@@ -128,7 +138,6 @@ public class ImportCollectionIT {
deleteBucket();
exportCollection( emApp1, "things" );
-
// create new second application, import the data from S3
final UUID appId2 = setup.getMgmtSvc().createApplication(
@@ -173,7 +182,7 @@ public class ImportCollectionIT {
Assert.assertEquals( 1, connected1.size() );
Assert.assertEquals( 1, connecting1.size() );
- // rest do not
+ // the rest rest do not have connections
EntityRef entity2 = importedThings.get(2);
Map connected2 = emApp2.getDictionaryAsMap(entity2, "connected_types");
@@ -244,7 +253,7 @@ public class ImportCollectionIT {
// import the updated things back into the first application, check that they've been updated
- importCollection( emApp1, "things" );
+ importCollection(emApp1, "things");
for ( UUID uuid : thingsMap.keySet() ) {
Entity entity = emApp1.get( uuid );
@@ -254,34 +263,75 @@ public class ImportCollectionIT {
/**
- * Test that an existing collection of entities can be updated
- * by doing an import of entities identified by UUIDs.
+ * Test that the types of incoming entities is ignored.
*/
@Test
- public void testCreateByImport() {
+ public void testImportWithWrongTypes() throws Exception {
+
+ // create an app with a collection of cats, export it to S3
+
+ String appName = "import-test-" + RandomStringUtils.randomAlphanumeric(10);
+ UUID appId = setup.getMgmtSvc().createApplication(organization.getUuid(), appName).getId();
+
+ Map<UUID, Entity> catsMap = new HashMap<>();
+ List<Entity> cats = new ArrayList<>();
+
+ EntityManager emApp = setup.getEmf().getEntityManager( appId );
+ createTestEntities( emApp, catsMap, cats, "cat");
+ exportCollection(emApp, "cats");
- // import from a JSON file stored locally on disk, with no UUID or type info
+ // import the cats data into a new collection called dogs in the default test app
- // check that entities were created
+ final EntityManager emDefaultApp = setup.getEmf().getEntityManager( applicationId );
+ importCollection( emDefaultApp, "dogs" );
+
+ // check that we now have a collection of dogs in the default test app
+
+ List<Entity> importedThings = emDefaultApp.getCollection(
+ emDefaultApp.getApplicationId(), "dogs", null, Level.ALL_PROPERTIES).getEntities();
+
+ assertTrue( !importedThings.isEmpty() );
+ assertEquals( 10, importedThings.size() );
}
- /**
- * Test that the types of incoming entities is ignored.
+ /**
+ * Simple import test but with multiple files.
*/
@Test
- public void testImportWithWrongTypes() {
+ public void testImportWithMultipleFiles() throws Exception {
- // import from a JSON file stored locally on disk, with incorrect type for collection
+ // create 10 applications each with collection of 10 things, export all to S3
- // check that entities were created with collection's type
+ Map<UUID, Entity> thingsMap = new HashMap<>();
+ List<Entity> things = new ArrayList<>();
+
+ for ( int i=0; i<10; i++) {
+ String appName = "import-test-" + i + RandomStringUtils.randomAlphanumeric(10);
+ UUID appId = setup.getMgmtSvc().createApplication(organization.getUuid(), appName).getId();
+ EntityManager emApp = setup.getEmf().getEntityManager( appId );
+ createTestEntities( emApp, thingsMap, things, "thing" );
+ exportCollection( emApp, "things" );
+ }
+
+ // import all those exports from S3 into the default test application
+
+ final EntityManager emDefaultApp = setup.getEmf().getEntityManager( applicationId );
+ importCollection( emDefaultApp, "things" );
+
+ // we should now have 100 Entities in the default app
+ List<Entity> importedThings = emDefaultApp.getCollection(
+ emDefaultApp.getApplicationId(), "things", null, Level.ALL_PROPERTIES).getEntities();
+
+ assertTrue( !importedThings.isEmpty() );
+ assertEquals( 100, importedThings.size() );
}
/**
- * Test that importing bad JSON will result in an informative error message.
+ * TODO: Test that importing bad JSON will result in an informative error message.
*/
@Test
public void testImportBadJson() {
@@ -292,16 +342,7 @@ public class ImportCollectionIT {
}
- /**
- * Simple import test but with multiple files.
- */
- @Test
- public void testImportWithMultipleFiles() {
-
- // create collection of things in first app, export them to S3
-
- // create collection of things in second app, export them to S3
- }
+ //---------------------------------------------------------------------------------------------
/**
@@ -340,6 +381,7 @@ public class ImportCollectionIT {
em.refreshIndex();
}
+
/**
* Call exportService to export the named collection to the configured S3 bucket.
*/
@@ -375,6 +417,7 @@ public class ImportCollectionIT {
}
}
+
/**
* Create test entities of a specified type.
* First two entities are connected.
@@ -385,6 +428,9 @@ public class ImportCollectionIT {
logger.debug("\n\nCreating new {} collection in application {}\n",
type, em.getApplication().getName() );
+ em.refreshIndex();
+
+ List<Entity> created = new ArrayList<>();
for ( int i = 0; i < 10; i++ ) {
final int count = i;
Entity e = em.create( type, new HashMap<String, Object>() {{
@@ -394,24 +440,18 @@ public class ImportCollectionIT {
}});
thingsMap.put(e.getUuid(), e);
things.add( e );
+ created.add( e );
}
// first two things are related to each other
- em.createConnection( new SimpleEntityRef( "thing", things.get(0).getUuid()),
- "related", new SimpleEntityRef( "thing", things.get(1).getUuid()));
- em.createConnection( new SimpleEntityRef( "thing", things.get(1).getUuid()),
- "related", new SimpleEntityRef( "thing", things.get(0).getUuid()));
+ em.createConnection( new SimpleEntityRef( type, created.get(0).getUuid()),
+ "related", new SimpleEntityRef( type, created.get(1).getUuid()));
+ em.createConnection( new SimpleEntityRef( type, created.get(1).getUuid()),
+ "related", new SimpleEntityRef( type, created.get(0).getUuid()));
em.refreshIndex();
}
- @After
- public void after() throws Exception {
- if(listener != null) {
- listener.stop();
- listener = null;
- }
- }
/**
* Delete the configured s3 bucket.
[15/16] incubator-usergrid git commit: Merge branch
'two-dot-o-import' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-import
Posted by sn...@apache.org.
Merge branch 'two-dot-o-import' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-import
Conflicts:
stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportTrackerTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/54d31cd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/54d31cd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/54d31cd3
Branch: refs/heads/two-dot-o-import
Commit: 54d31cd3680debbc1cf817f71586754173cdb3f6
Parents: 7e1c83d 57ae816
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Feb 5 09:25:48 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Feb 5 09:25:48 2015 -0500
----------------------------------------------------------------------
.../importer/FileImportStatistics.java | 319 ---------------
.../management/importer/FileImportTracker.java | 313 ++++++++++++++
.../management/importer/ImportServiceImpl.java | 35 +-
.../importer/FileImportStatisticsTest.java | 364 ----------------
.../importer/FileImportTrackerTest.java | 410 +++++++++++++++++++
5 files changed, 739 insertions(+), 702 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54d31cd3/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
index 0000000,8a01259..cda01a6
mode 000000,100644..100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
@@@ -1,0 -1,333 +1,313 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ package org.apache.usergrid.management.importer;
+
+
-import java.util.UUID;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
++import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+ import org.apache.usergrid.persistence.EntityManager;
++import org.apache.usergrid.persistence.EntityManagerFactory;
+ import org.apache.usergrid.persistence.entities.FailedImportConnection;
+ import org.apache.usergrid.persistence.entities.FailedImportEntity;
+ import org.apache.usergrid.persistence.entities.FileImport;
-import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
+ import org.apache.usergrid.persistence.exceptions.PersistenceException;
+
++import java.util.concurrent.Semaphore;
++import java.util.concurrent.atomic.AtomicInteger;
++import java.util.concurrent.atomic.AtomicLong;
++
+
+ /**
- * Statistics used to track a file import. Only 1 instance of this class should exist per file imported in the cluster.
- * There is a direct 1-1 mapping of the statistics provided here and the file import status. This class is threadsafe to
- * be used across multiple threads.
++ * Statistics used to track a file import. Only 1 instance of this class should exist
++ * per file imported in the cluster. There is a direct 1-1 mapping of the statistics provided
++ * here and the file import status. This class is thread-safe to be used across multiple threads.
+ */
+ public class FileImportTracker {
+
- private static final String ERROR_MESSAGE = "Failed to import some data. See the import counters and errors.";
-
++ private static final String ERROR_MESSAGE =
++ "Failed to import some data. See the import counters and errors.";
+
+ private static final String ERRORS_CONNECTION_NAME = "errors";
+
+ private final AtomicLong entitiesWritten = new AtomicLong( 0 );
+ private final AtomicLong entitiesFailed = new AtomicLong( 0 );
+ private final AtomicLong connectionsWritten = new AtomicLong( 0 );
+ private final AtomicLong connectionsFailed = new AtomicLong( 0 );
+ private final AtomicInteger cachedOperations = new AtomicInteger( 0 );
+
+ private final Semaphore writeSemaphore = new Semaphore( 1 );
+
+ private final FileImport fileImport;
- private final EntityManager entityManager;
++ private final EntityManagerFactory emf;
+ private final int flushCount;
+
+
+ /**
- * Create an instance to track counters. Note that when this instance is created, it will attempt to load it's state
- * from the entity manager. In the case of using this when resuming, be sure you begin processing where the system thinks
- * it has left off.
++ * Create an instance to track counters. Note that when this instance is created, it will
++ * attempt to load it's state from the entity manager. In the case of using this when resuming,
++ * be sure you begin processing where the system thinks * it has left off.
+ *
- * @param entityManager The entity manager that will hold these entities.
- * @param fileImportId The uuid of the fileImport
++ * @param emf Entity Manager Factory
++ * @param fileImport File Import Entity
+ * @param flushCount The number of success + failures to accumulate before flushing
+ */
- public FileImportTracker( final EntityManager entityManager, final UUID fileImportId, final int flushCount ) {
- this.entityManager = entityManager;
++ public FileImportTracker(
++ final EntityManagerFactory emf, final FileImport fileImport, final int flushCount ) {
++
++ this.emf = emf;
+ this.flushCount = flushCount;
- this.fileImport = getFileImport( fileImportId );
++ this.fileImport = fileImport;
+
+ this.entitiesWritten.addAndGet( fileImport.getImportedEntityCount() );
+ this.entitiesFailed.addAndGet( fileImport.getFailedEntityCount() );
+
+ this.connectionsWritten.addAndGet( fileImport.getImportedConnectionCount() );
+ this.connectionsFailed.addAndGet( fileImport.getFailedConnectionCount() );
+ }
+
+
+ /**
+ * Invoke when an entity has been successfully written
+ */
+ public void entityWritten() {
+ entitiesWritten.incrementAndGet();
+ maybeFlush();
+ }
+
+
-
-
+ /**
+ * Invoke when an entity fails to write correctly
+ */
+
+ public void entityFailed( final String message ) {
+ entitiesFailed.incrementAndGet();
+
-
+ FailedImportEntity failedImportEntity = new FailedImportEntity();
+ failedImportEntity.setErrorMessage( message );
+
+ try {
++ EntityManager entityManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
+ failedImportEntity = entityManager.create( failedImportEntity );
+ entityManager.createConnection( fileImport, ERRORS_CONNECTION_NAME, failedImportEntity );
+ }
+ catch ( Exception e ) {
+ throw new PersistenceException( "Unable to save failed entity import message", e );
+ }
+ maybeFlush();
+ }
+
+
+ /**
+ * Invoked when a connection is written
+ */
+ public void connectionWritten() {
+ connectionsWritten.incrementAndGet();
+ maybeFlush();
+ }
+
+
+ /**
+ * Invoked when a connection cannot be written
+ */
+ public void connectionFailed( final String message ) {
+ connectionsFailed.incrementAndGet();
+
+
+ FailedImportConnection failedImportConnection = new FailedImportConnection();
+ failedImportConnection.setErrorMessage( message );
+
+ try {
++ EntityManager entityManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
+ failedImportConnection = entityManager.create( failedImportConnection );
+ entityManager.createConnection( fileImport, ERRORS_CONNECTION_NAME, failedImportConnection );
+ }
+ catch ( Exception e ) {
+ throw new PersistenceException( "Unable to save failed entity import message", e );
+ }
+ maybeFlush();
+ }
+
+
+ /**
+ * Invoke when the file is completed processing
+ */
+ public void complete() {
+
+ final long failed = entitiesFailed.get() + connectionsFailed.get();
+
+ final FileImport.State state;
+ final String message;
+
+ if ( failed > 0 ) {
+ state = FileImport.State.FAILED;
+ message = ERROR_MESSAGE;
+ }
+ else {
+ state = FileImport.State.FINISHED;
+ message = null;
+ }
+
+ updateFileImport( state, message );
+ }
+
+
+ /**
+ * Invoke when we halt the import with a fatal error that cannot be recovered.
+ */
+ public void fatal( final String message ) {
+
+ updateFileImport( FileImport.State.FAILED, message );
+ }
+
+
+ /**
- * Return the total number of successful imports + failed imports. Can be used in resume. Note that this reflects
- * the counts last written to cassandra when this instance was created + any processing
++ * Return the total number of successful imports + failed imports.
++ * Can be used in resume. Note that this reflects the counts last written
++ * to cassandra when this instance was created + any processing
+ */
+ public long getTotalEntityCount() {
+ return getEntitiesWritten() + getEntitiesFailed();
+ }
+
+
+ /**
+ * Get the total number of failed + successful connections
+ * @return
+ */
+ public long getTotalConnectionCount(){
+ return getConnectionsFailed() + getConnectionsWritten();
+ }
+
+
+ /**
+ * Returns true if we should stop processing. We use fail fast logic, so after the first
+ * failure this will return true.
+ */
+ public boolean shouldStopProcessingEntities() {
+ return entitiesFailed.get() > 0;
+ }
+
+
+ /**
+ * Returns true if we should stop processing. We use fail fast logic, so after the first
+ * failure this will return true.
+ */
+ public boolean shouldStopProcessingConnections() {
+ return connectionsFailed.get() > 0;
+ }
+
+ /**
+ * Get the number of entities written
+ * @return
+ */
+ public long getEntitiesWritten() {
+ return entitiesWritten.get();
+ }
+
+
+ /**
+ * Get the number of failed entities
+ * @return
+ */
+ public long getEntitiesFailed() {
+ return entitiesFailed.get();
+ }
+
+
+ /**
+ * Get the number of connections written
+ * @return
+ */
+ public long getConnectionsWritten() {
+ return connectionsWritten.get();
+ }
+
+
+ /**
+ * Get the number of connections failed
+ * @return
+ */
+ public long getConnectionsFailed() {
+ return connectionsFailed.get();
+ }
+
+ private void maybeFlush() {
+ final int count = cachedOperations.incrementAndGet();
+
+ //no op
+ if ( count < flushCount ) {
+ return;
+ }
+
+ //another thread is writing, no op, just return
+ if ( !writeSemaphore.tryAcquire() ) {
+ return;
+ }
+
+ final long failed = entitiesFailed.get();
+ final long written = entitiesWritten.get();
+ final String message;
+
+ if ( failed > 0 ) {
- message = "Failed to import " + failed + " entities. Successfully imported " + written + " entities";
++ message = "Failed to import " + failed
++ + " entities. Successfully imported " + written + " entities";
+ }
+ else {
+ message = "Successfully imported " + written + " entities";
+ }
+
+ updateFileImport( FileImport.State.STARTED, message );
+ cachedOperations.addAndGet( flushCount * -1 );
+ writeSemaphore.release();
+ }
+
+
+ /**
+ * Update the file import status with the provided messages
+ *
+ * @param state The state to set into the import
+ * @param message The message to set
+ */
+ private void updateFileImport( final FileImport.State state, final String message ) {
+
+ try {
+
+
+ final long writtenEntities = entitiesWritten.get();
+ final long failedEntities = entitiesFailed.get();
+
+ final long writtenConnections = connectionsFailed.get();
+ final long failedConnections = connectionsFailed.get();
+
+
+ fileImport.setImportedEntityCount( writtenEntities );
+ fileImport.setFailedEntityCount( failedEntities );
+ fileImport.setImportedConnectionCount( writtenConnections );
+ fileImport.setFailedConnectionCount( failedConnections );
+ fileImport.setState( state );
+ fileImport.setErrorMessage( message );
+
++ EntityManager entityManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
+ entityManager.update( fileImport );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to persist complete state", e );
+ }
+ }
-
-
- /**
- * Get the FileImport by uuid and return it
- *
- * @throws EntityNotFoundException if we can't find the file import with the given uuid
- */
- private FileImport getFileImport( final UUID fileImportId ) {
-
- final FileImport fileImport;
-
- try {
- fileImport = entityManager.get( fileImportId, FileImport.class );
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Unable to load fileImport with id " + fileImportId, e );
- }
-
- if ( fileImport == null ) {
- throw new EntityNotFoundException( "Could not file FileImport with id " + fileImportId );
- }
-
- return fileImport;
- }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54d31cd3/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index 1fea660,5646f46..b38aa4b
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@@ -697,8 -700,8 +697,6 @@@ public class ImportServiceImpl implemen
* @param collectionFile the file being validated
* @param rootEm the Entity Manager for the Management application
* @param fileImport the file import entity
-- * @return
-- * @throws Exception
*/
private boolean isValidJSON(File collectionFile, EntityManager rootEm, FileImport fileImport)
throws Exception {
@@@ -761,12 -763,11 +759,11 @@@
new JsonEntityParserObservable(jp, em, rootEm, fileImport, entitiesOnly);
final Observable<WriteEvent> entityEventObservable = Observable.create(jsonObservableEntities);
- //flush every 100 entities
- final FileImportTracker tracker = new FileImportTracker( em, fileImport.getUuid(), 100 );
- //truncate due to RX api
+ // flush every 100 entities
- final FileImportStatistics statistics = new FileImportStatistics( emf, fileImport, 100 );
-
++ final FileImportTracker tracker = new FileImportTracker( emf, fileImport, 100 );
+ // truncate due to RX api
- final int entityNumSkip = (int)statistics.getTotalEntityCount();
- final int connectionNumSkip = (int)statistics.getTotalConnectionCount();
+ final int entityNumSkip = (int)tracker.getTotalEntityCount();
+ final int connectionNumSkip = (int)tracker.getTotalConnectionCount();
// function to execute for each write event
@@@ -824,10 -832,10 +821,10 @@@
jp.close();
- //flush the job statistics
- tracker.complete();
+ logger.debug("\n\nimportEntitiesFromFile(): Wrote others\n");
- //flush the job statistics
- statistics.complete();
- logger.debug("\n\nWrote others\n");
++ // flush the job statistics
++ tracker.complete();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54d31cd3/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportTrackerTest.java
----------------------------------------------------------------------
diff --cc stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportTrackerTest.java
index 0000000,0a58602..2978c8b
mode 000000,100644..100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportTrackerTest.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/FileImportTrackerTest.java
@@@ -1,0 -1,393 +1,410 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ package org.apache.usergrid.management.importer;
+
+
+ import java.util.List;
+ import java.util.UUID;
+
++import org.apache.usergrid.corepersistence.util.CpNamingUtils;
++import org.apache.usergrid.persistence.EntityManagerFactory;
+ import org.junit.Test;
+ import org.mockito.ArgumentCaptor;
+ import org.mockito.invocation.InvocationOnMock;
+ import org.mockito.stubbing.Answer;
+
+ import org.apache.usergrid.persistence.EntityManager;
+ import org.apache.usergrid.persistence.entities.FailedImport;
+ import org.apache.usergrid.persistence.entities.FailedImportConnection;
+ import org.apache.usergrid.persistence.entities.FailedImportEntity;
+ import org.apache.usergrid.persistence.entities.FileImport;
+ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertFalse;
+ import static org.junit.Assert.assertNull;
+ import static org.junit.Assert.assertSame;
+ import static org.junit.Assert.assertTrue;
+ import static org.mockito.Matchers.any;
+ import static org.mockito.Matchers.eq;
+ import static org.mockito.Matchers.same;
+ import static org.mockito.Mockito.mock;
+ import static org.mockito.Mockito.times;
+ import static org.mockito.Mockito.verify;
+ import static org.mockito.Mockito.when;
+
+
+ public class FileImportTrackerTest {
+
+ @Test
+ public void testSuccess() throws Exception {
+
++ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
+ final EntityManager em = mock( EntityManager.class );
++ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
+
+ final UUID importFileId = UUIDGenerator.newTimeUUID();
+
-
+ final FileImport fileImport = new FileImport();
+ fileImport.setUuid( importFileId );
+
+ when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
+
-
- final FileImportTracker fileImportTracker = new FileImportTracker( em, importFileId, 1000 );
++ final FileImportTracker fileImportTracker = new FileImportTracker( emf, fileImport, 1000 );
+
+ final long expectedCount = 100;
+
+ for ( long i = 0; i < expectedCount; i++ ) {
+ fileImportTracker.entityWritten();
+ }
+
+
+ fileImportTracker.complete();
+
+
+ ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class );
+
+ verify( em ).update( savedFileImport.capture() );
+
+ final FileImport updated = savedFileImport.getValue();
+
+ assertSame( "Same instance should be updated", fileImport, updated );
+
+
+ assertEquals( "Same count expected", expectedCount, updated.getImportedEntityCount() );
+
+ assertNull( updated.getErrorMessage() );
+ }
+
+
+ @Test
+ public void testBoth() throws Exception {
+
++ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
+ final EntityManager em = mock( EntityManager.class );
++ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
+
+ final UUID importFileId = UUIDGenerator.newTimeUUID();
+
+
+ final FileImport fileImport = new FileImport();
- fileImport.setUuid( importFileId );
-
- when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
++ fileImport.setUuid(importFileId);
+
+ //mock up returning the FailedEntityImport instance after save is invoked.
+
+ when( em.create( any( FailedImportEntity.class ) ) ).thenAnswer( new Answer<FailedImportEntity>() {
+ @Override
+ public FailedImportEntity answer( final InvocationOnMock invocation ) throws Throwable {
+ return ( FailedImportEntity ) invocation.getArguments()[0];
+ }
+ } );
+
- final FileImportTracker fileImportTracker = new FileImportTracker( em, importFileId, 1000 );
++ final FileImportTracker fileImportTracker = new FileImportTracker( emf, fileImport, 1000 );
+
+ final long expectedSuccess = 100;
+
+ for ( long i = 0; i < expectedSuccess; i++ ) {
+ fileImportTracker.entityWritten();
+ }
+
+ final int expectedFails = 10;
+
+ for ( int i = 0; i < expectedFails; i++ ) {
+ fileImportTracker.entityFailed( "Failed to write entity " + i );
+ }
+
+
+ fileImportTracker.complete();
+
+
+ ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class );
+
+ verify( em ).update( savedFileImport.capture() );
+
+ final FileImport updated = savedFileImport.getValue();
+
+ assertSame( "Same instance should be updated", fileImport, updated );
+
+ assertEquals( "Same count expected", expectedSuccess, updated.getImportedEntityCount() );
+
+ assertEquals( "Same fail expected", expectedFails, updated.getFailedEntityCount() );
+
- assertEquals( "Correct error message", "Failed to import some data. See the import counters and errors.",
++ assertEquals( "Correct error message",
++ "Failed to import some data. See the import counters and errors.",
+ updated.getErrorMessage() );
+
+ //TODO get the connections from the file import
+
- ArgumentCaptor<FailedImportEntity> failedEntities = ArgumentCaptor.forClass( FailedImportEntity.class );
++ ArgumentCaptor<FailedImportEntity> failedEntities =
++ ArgumentCaptor.forClass( FailedImportEntity.class );
+
+ verify( em, times( expectedFails ) )
+ .createConnection( same( fileImport ), eq( "errors" ), failedEntities.capture() );
+
+ //now check all our arguments
+
+ final List<FailedImportEntity> args = failedEntities.getAllValues();
+
+ assertEquals( "Same number of error connections created", expectedFails, args.size() );
+
+
+ for ( int i = 0; i < expectedFails; i++ ) {
+
+ final FailedImportEntity failedImport = args.get( i );
+
- assertEquals( "Same message expected", "Failed to write entity " + i, failedImport.getErrorMessage() );
++ assertEquals( "Same message expected",
++ "Failed to write entity " + i, failedImport.getErrorMessage() );
+ }
+ }
+
+
+ @Test
+ public void explicitFail() throws Exception {
+
++ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
+ final EntityManager em = mock( EntityManager.class );
++ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
+
+ final UUID importFileId = UUIDGenerator.newTimeUUID();
+
+
+ final FileImport fileImport = new FileImport();
+ fileImport.setUuid( importFileId );
+
+ when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
+
+
- final FileImportTracker fileImportTracker = new FileImportTracker( em, importFileId, 1000 );
++ final FileImportTracker fileImportTracker = new FileImportTracker( emf, fileImport, 1000 );
+
+ final long expectedCount = 100;
+
+ for ( long i = 0; i < expectedCount; i++ ) {
+ fileImportTracker.entityWritten();
+ }
+
+
+ fileImportTracker.fatal( "Something bad happened" );
+
+
+ ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class );
+
+ verify( em ).update( savedFileImport.capture() );
+
+ final FileImport updated = savedFileImport.getValue();
+
+ assertSame( "Same instance should be updated", fileImport, updated );
+
+
+ assertEquals( "Same count expected", expectedCount, updated.getImportedEntityCount() );
+
+ assertEquals( "Fail count is 0", 0, updated.getFailedEntityCount() );
+
+ assertEquals( "Correct expected message", "Something bad happened", updated.getErrorMessage() );
+
+ assertEquals( "Expected failed state", FileImport.State.FAILED, updated.getState() );
+ }
+
+
+ @Test
+ public void testAutoFlushSuccess() throws Exception {
+
++ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
+ final EntityManager em = mock( EntityManager.class );
++ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
+
+ final UUID importFileId = UUIDGenerator.newTimeUUID();
+
+
+ final FileImport fileImport = new FileImport();
+ fileImport.setUuid( importFileId );
+
+ when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
+
+ //mock up returning the FailedEntityImport instance after save is invoked.
+
+
+ when( em.create( any( FailedImportConnection.class ) ) ).thenAnswer( new Answer<Object>() {
+ @Override
+ public Object answer( final InvocationOnMock invocation ) throws Throwable {
+ return invocation.getArguments()[0];
+ }
+ } );
+
+ final int expectedSuccess = 100;
+ final int expectedFails = 100;
+ final int expectedConnectionSuccess = 100;
+ final int expectedConnectionFails = 100;
+
+ final int expectedFlushCount = 2;
- final int flushSize = ( expectedFails + expectedFails + expectedConnectionSuccess + expectedConnectionFails )
++ final int flushSize =
++ ( expectedFails + expectedFails + expectedConnectionSuccess + expectedConnectionFails )
+ / expectedFlushCount;
+
+ //set this to 1/2, so that we get saved twice
- final FileImportTracker fileImportTracker = new FileImportTracker( em, importFileId, flushSize );
++ final FileImportTracker fileImportTracker = new FileImportTracker( emf, fileImport, flushSize );
+
+
+ for ( long i = 0; i < expectedSuccess; i++ ) {
+ fileImportTracker.entityWritten();
+ }
+
+
+ for ( int i = 0; i < expectedFails; i++ ) {
+ fileImportTracker.entityFailed( "Failed to write entity " + i );
+ }
+
+
+ for ( long i = 0; i < expectedConnectionSuccess; i++ ) {
+ fileImportTracker.connectionWritten();
+ }
+
+
+ for ( int i = 0; i < expectedConnectionFails; i++ ) {
+ fileImportTracker.connectionFailed( "Failed to write connection " + i );
+ }
+
+
+ fileImportTracker.complete();
+
+
+ ArgumentCaptor<FileImport> savedFileImport = ArgumentCaptor.forClass( FileImport.class );
+
+ verify( em, times( expectedFlushCount + 1 ) ).update( savedFileImport.capture() );
+
+ final FileImport updated = savedFileImport.getAllValues().get( 2 );
+
+ assertSame( "Same instance should be updated", fileImport, updated );
+
+ assertEquals( "Same count expected", expectedSuccess, updated.getImportedEntityCount() );
+
+ assertEquals( "Same fail expected", expectedFails, updated.getFailedEntityCount() );
+
+ assertEquals( "Same connection count expected", expectedConnectionSuccess,
+ updated.getImportedConnectionCount() );
+
+ assertEquals( "Same connection error count expected", expectedConnectionFails,
+ updated.getFailedConnectionCount() );
+
- assertEquals( "Correct error message", "Failed to import some data. See the import counters and errors.",
++ assertEquals( "Correct error message",
++ "Failed to import some data. See the import counters and errors.",
+ updated.getErrorMessage() );
+
+ //TODO get the connections from the file import
+
+ ArgumentCaptor<FailedImport> failedEntities = ArgumentCaptor.forClass( FailedImport.class );
+
+ verify( em, times( expectedFails + expectedConnectionFails ) )
+ .createConnection( same( fileImport ), eq( "errors" ), failedEntities.capture() );
+
+ //now check all our arguments
+
+ final List<FailedImport> args = failedEntities.getAllValues();
+
- assertEquals( "Same number of error connections created", expectedFails + expectedConnectionFails,
- args.size() );
++ assertEquals( "Same number of error connections created",
++ expectedFails + expectedConnectionFails, args.size() );
+
+
+ for ( int i = 0; i < expectedFails; i++ ) {
+
+ final FailedImport failedImport = args.get( i );
+
- assertEquals( "Same message expected", "Failed to write entity " + i, failedImport.getErrorMessage() );
++ assertEquals( "Same message expected",
++ "Failed to write entity " + i, failedImport.getErrorMessage() );
+ }
+
+ for ( int i = expectedFails; i < expectedConnectionFails; i++ ) {
+
+ final FailedImport failedImport = args.get( i );
+
- assertEquals( "Same message expected", "Failed to write connection " + i, failedImport.getErrorMessage() );
++ assertEquals( "Same message expected",
++ "Failed to write connection " + i, failedImport.getErrorMessage() );
+ }
+ }
+
+
+ @Test
+ public void loadingExistingState() throws Exception {
+
++ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
+ final EntityManager em = mock( EntityManager.class );
++ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
+
+ final UUID importFileId = UUIDGenerator.newTimeUUID();
+
+
+ final FileImport fileImport = new FileImport();
+ fileImport.setUuid( importFileId );
+ fileImport.setImportedEntityCount( 1 );
+ fileImport.setFailedEntityCount( 2 );
+ fileImport.setImportedConnectionCount( 3 );
+ fileImport.setFailedConnectionCount( 4 );
+
+ when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
+
+ //mock up returning the FailedEntityImport instance after save is invoked.
+
- FileImportTracker statistics = new FileImportTracker( em, importFileId, 100 );
++ FileImportTracker statistics = new FileImportTracker( emf, fileImport, 100 );
+
+ assertEquals( 1, statistics.getEntitiesWritten() );
+ assertEquals( 2, statistics.getEntitiesFailed() );
+
+ assertEquals( 3, statistics.getTotalEntityCount() );
+
+ assertEquals( 3, statistics.getConnectionsWritten() );
+ assertEquals( 4, statistics.getConnectionsFailed() );
+
+ assertEquals( 7, statistics.getTotalConnectionCount() );
+ }
+
+
+ @Test
+ public void failFast() throws Exception {
+
++ final EntityManagerFactory emf = mock( EntityManagerFactory.class );
+ final EntityManager em = mock( EntityManager.class );
++ when( emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID ) ).thenReturn( em );
+
+ final UUID importFileId = UUIDGenerator.newTimeUUID();
+
+
+ final FileImport fileImport = new FileImport();
+
+ when( em.get( importFileId, FileImport.class ) ).thenReturn( fileImport );
+
+ //mock up returning the FailedEntityImport instance after save is invoked.
+
- FileImportTracker statistics = new FileImportTracker( em, importFileId, 100 );
++ FileImportTracker statistics = new FileImportTracker( emf, fileImport, 100 );
+
+
+ assertFalse( statistics.shouldStopProcessingEntities() );
+
+ assertFalse( statistics.shouldStopProcessingConnections() );
+
+
+ statistics.entityFailed( "test fail" );
+
+ assertTrue("We shouldn't process after a failure", statistics.shouldStopProcessingEntities());
+
+ statistics.connectionFailed( "test fail" );
+
+ assertTrue( "We shouldn't process after a failure", statistics.shouldStopProcessingConnections() );
+ }
+ }
+
[08/16] incubator-usergrid git commit: Use example URL instead
Posted by sn...@apache.org.
Use example URL instead
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/198f4795
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/198f4795
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/198f4795
Branch: refs/heads/two-dot-o-import
Commit: 198f4795af99a466d2657bc078868d5a3312df98
Parents: bfdacb3
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jan 28 10:51:54 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jan 28 10:51:54 2015 -0500
----------------------------------------------------------------------
.../samples/notifications/Usergrid.Notifications/MainPage.xaml.cs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/198f4795/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
index 6d4c4ec..a550954 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
@@ -44,7 +44,7 @@ namespace Usergrid.Notifications
{
this.InitializeComponent();
//TODO: change me to your server
- serverUrl = "https://apigee-internal-prod.mars.apigee.net";
+ serverUrl = "https://usergrid-push.example.com";
//TODO: change me to your org
org = "ugvalidate";
//TODO: change me to your app
[03/16] incubator-usergrid git commit: Revert "remove lexer"
Posted by sn...@apache.org.
Revert "remove lexer"
This reverts commit df29a99c4218a39269151a8773c0b0e051507650.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/eafdf633
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/eafdf633
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/eafdf633
Branch: refs/heads/two-dot-o-import
Commit: eafdf633b7500d9b224b66046aea3dafb466627d
Parents: df29a99
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Jan 27 09:19:11 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Jan 27 09:19:11 2015 -0700
----------------------------------------------------------------------
.gitignore | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eafdf633/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 6f8e854..522f674 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,8 @@
.DS_Store
.AppleDouble
.LSOverride
+stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
+stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
target
@@ -39,6 +41,8 @@ stack/corepersistence/graph/nbactions.xml
stack/corepersistence/model/nbactions.xml
stack/corepersistence/nbactions.xml
stack/corepersistence/queryindex/nbactions.xml
+stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
+stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
/stack/corepersistence/perftest2/src/main/resources/finefoods.txt
/stack/corepersistence/perftest1/src/main/resources/finefoods.txt
/stack/corepersistence/perftest2/nbactions.xml
@@ -76,6 +80,8 @@ portal/js/templates.js
/release/target/
/portal/seleniumLog.txt
+/stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
+/stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
!/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
*.iml
sdks/dotnet/samples/notifications/packages/*
[06/16] incubator-usergrid git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
apache_usergrid/two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into apache_usergrid/two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6ba340a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6ba340a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6ba340a3
Branch: refs/heads/two-dot-o-import
Commit: 6ba340a360a605826fcc2a49cbbac15dc1cf814e
Parents: 373248b 6da8119
Author: Rod Simpson <ro...@apigee.com>
Authored: Tue Jan 27 13:24:54 2015 -0600
Committer: Rod Simpson <ro...@apigee.com>
Committed: Tue Jan 27 13:24:54 2015 -0600
----------------------------------------------------------------------
.gitignore | 4 +-
portal/js/global/ug-service.js | 67 +++----
portal/js/push/push-config-controller.js | 18 +-
portal/js/push/push-config.html | 50 +++++
.../Usergrid.Notifications/App.xaml | 27 +++
.../Usergrid.Notifications/App.xaml.cs | 150 ++++++++++++++
.../Assets/Logo.scale-240.png | Bin 0 -> 2516 bytes
.../Assets/SmallLogo.scale-240.png | Bin 0 -> 753 bytes
.../Assets/SplashScreen.scale-240.png | Bin 0 -> 14715 bytes
.../Assets/Square71x71Logo.scale-240.png | Bin 0 -> 1122 bytes
.../Assets/StoreLogo.scale-240.png | Bin 0 -> 2200 bytes
.../Assets/WideLogo.scale-240.png | Bin 0 -> 4530 bytes
.../Client/EntityResponse.cs | 80 ++++++++
.../Client/IUsergridClient.cs | 123 ++++++++++++
.../Usergrid.Notifications/Client/PushClient.cs | 153 +++++++++++++++
.../Usergrid.Notifications/Client/Usergrid.cs | 120 ++++++++++++
.../Usergrid.Notifications/MainPage.xaml | 38 ++++
.../Usergrid.Notifications/MainPage.xaml.cs | 127 ++++++++++++
.../Package.StoreAssociation.xml | 194 +++++++++++++++++++
.../Usergrid.Notifications/Package.appxmanifest | 53 +++++
.../Properties/AssemblyInfo.cs | 29 +++
.../Usergrid.Notifications.csproj | 142 ++++++++++++++
.../Usergrid.Notifications/packages.config | 25 +++
.../notifications/packages/repositories.config | 4 +
.../usergrid/persistence/entities/Notifier.java | 31 ++-
.../persistence/queue/impl/QueueScopeImpl.java | 4 -
stack/services/pom.xml | 7 +-
.../notifications/ProviderAdapterFactory.java | 5 +
.../impl/ApplicationQueueManagerImpl.java | 8 +-
.../wns/TranslatedNotification.java | 50 +++++
.../services/notifications/wns/WNSAdapter.java | 163 ++++++++++++++++
31 files changed, 1609 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6ba340a3/portal/js/global/ug-service.js
----------------------------------------------------------------------
[13/16] incubator-usergrid git commit: Merge branch
'two-dot-o-import' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-import
Posted by sn...@apache.org.
Merge branch 'two-dot-o-import' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-import
Conflicts:
stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1d0e1a19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1d0e1a19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1d0e1a19
Branch: refs/heads/two-dot-o-import
Commit: 1d0e1a19fc4791d531fb7124eae80cc2757e7f08
Parents: b47904e aafa3d4
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Feb 4 14:56:49 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Feb 4 14:56:49 2015 -0500
----------------------------------------------------------------------
.../persistence/entities/FailedImport.java | 64 +++++
.../entities/FailedImportConnection.java | 41 ++++
.../entities/FailedImportEntity.java | 44 ++++
.../persistence/entities/FileImport.java | 32 +++
.../exceptions/PersistenceException.java | 2 +-
stack/pom.xml | 11 +-
.../importer/FileImportStatistics.java | 240 ++++++++++++++++---
.../management/importer/ImportServiceImpl.java | 161 +++++--------
.../importer/FileImportStatisticsTest.java | 237 +++++++++++++++++-
9 files changed, 691 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1d0e1a19/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index 2054804,238c0ab..936e09b
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@@ -657,37 -651,8 +657,8 @@@ public class ImportServiceImpl implemen
EntityManager targetEm = emf.getEntityManager(targetAppId);
logger.debug(" importing into app {} file {}", targetAppId.toString(), file.getAbsolutePath());
- importEntitiesFromFile(file, targetEm, emManagementApp, fileImport );
+ importEntitiesFromFile( collectionName, file, targetEm, emManagementApp, fileImport );
- // TODO: fix the resume on error feature
-
- // // in case of resume, retrieve the last updated UUID for this file
- // String lastUpdatedUUID = fileImport.getLastUpdatedUUID();
- //
- // // this handles partially completed files by updating entities from the point of failure
- // if (!lastUpdatedUUID.equals(" ")) {
- //
- // // go till the last updated entity
- // while (!jp.getText().equals(lastUpdatedUUID)) {
- // jp.nextToken();
- // }
- //
- // // skip the last one and start from the next one
- // while (!(jp.getCurrentToken() == JsonToken.END_OBJECT
- // && jp.nextToken() == JsonToken.START_OBJECT)) {
- // jp.nextToken();
- // }
- // }
- //
- // // get to start of an object i.e next entity.
- // while (jp.getCurrentToken() != JsonToken.START_OBJECT) {
- // jp.nextToken();
- // }
- //
- // while (jp.nextToken() != JsonToken.END_ARRAY) {
- // importEntitiesFromFile(jp, targetEm, emManagementApp, fileImport, jobExecution);
- // }
- // jp.close();
// Updates the state of file import job
if (!fileImport.getState().toString().equals("FAILED")) {
@@@ -792,85 -756,86 +763,86 @@@
// observable that parses JSON and emits write events
JsonParser jp = getJsonParserForFile(file);
+
+ //TODO, move the json parser into the observable creation so that open/close happens automatcially within the stream
+
final JsonEntityParserObservable jsonObservableEntities =
- new JsonEntityParserObservable(jp, em, rootEm, fileImport, entitiesOnly);
+ new JsonEntityParserObservable(jp, em, rootEm, collectionName, fileImport, entitiesOnly);
final Observable<WriteEvent> entityEventObservable = Observable.create(jsonObservableEntities);
+ //flush every 100 entities
+ final FileImportStatistics statistics = new FileImportStatistics( em, fileImport.getUuid(), 100 );
+ //truncate due to RX api
+ final int entityNumSkip = (int)statistics.getTotalEntityCount();
+ final int connectionNumSkip = (int)statistics.getTotalConnectionCount();
+
// function to execute for each write event
- //TODO: job execution no longer needed due to having queueMessage.
+
+ /**
+ * Function that invokes the work of the event.
+ */
final Action1<WriteEvent> doWork = new Action1<WriteEvent>() {
@Override
- public void call(WriteEvent writeEvent) {
- writeEvent.doWrite(em,fileImport);
+ public void call( WriteEvent writeEvent ) {
+ writeEvent.doWrite( em, fileImport, statistics );
}
};
- // final AtomicLong entityCounter = new AtomicLong();
- // final AtomicLong eventCounter = new AtomicLong();
+
+
+
+
// start parsing JSON
- entityEventObservable.parallel(new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
+ //only take while our stats tell us we should continue processing
+ //potentially skip the first n if this is a resume operation
+ entityEventObservable.takeWhile( new Func1<WriteEvent, Boolean>() {
@Override
- public Observable<WriteEvent> call(Observable<WriteEvent> entityWrapperObservable) {
- // TODO: need to fixed so that number of entities created can be counted correctly and
- // TODO: also update last updated UUID for fileImport which is a must for resume-ability
-
- // return entityWrapperObservable.doOnNext(doWork).doOnNext(new Action1<WriteEvent>() {
- //
- // @Override
- // public void call(WriteEvent writeEvent) {
- // if (!(writeEvent instanceof EntityEvent)) {
- // final long val = eventCounter.incrementAndGet();
- // if(val % 50 == 0) {
- // jobExecution.heartbeat();
- // }
- // return;
- // }
- //
- // final long value = entityCounter.incrementAndGet();
- // if (value % 2000 == 0) {
- // try {
- // logger.error("UUID = {} value = {}",
- // ((EntityEvent) writeEvent).getEntityUuid().toString(),
- // value );
- // fileImport.setLastUpdatedUUID(
- // ((EntityEvent) writeEvent).getEntityUuid().toString());
- // //checkpoint the UUID here.
- // rootEm.update(fileImport);
- // } catch(Exception ex) {}
- // }
- // if(value % 100 == 0) {
- // logger.error("heartbeat sent by " + fileImport.getFileName());
- // jobExecution.heartbeat();
- // }
- // }
- // }
- // );
- return entityWrapperObservable.doOnNext(doWork);
+ public Boolean call( final WriteEvent writeEvent ) {
+ return !statistics.shouldStopProcessingEntities();
}
- }, Schedulers.io()).toBlocking().last();
+ } ).skip( entityNumSkip ).parallel( new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
+ @Override
+ public Observable<WriteEvent> call( Observable<WriteEvent> entityWrapperObservable ) {
+
+
+ return entityWrapperObservable.doOnNext( doWork );
+ }
+ }, Schedulers.io() ).toBlocking().last();
+
+
jp.close();
- logger.debug("\n\nWrote entities\n");
+ logger.debug("\n\nimportEntitiesFromFile(): Wrote entities\n");
// now do other stuff: connections and dictionaries
entitiesOnly = false;
// observable that parses JSON and emits write events
jp = getJsonParserForFile(file);
+
-
final JsonEntityParserObservable jsonObservableOther =
- new JsonEntityParserObservable( jp, em, rootEm, fileImport, entitiesOnly );
- final Observable<WriteEvent> otherEventObservable = Observable.create( jsonObservableOther );
+ new JsonEntityParserObservable(jp, em, rootEm, collectionName, fileImport, entitiesOnly);
+ final Observable<WriteEvent> otherEventObservable = Observable.create(jsonObservableOther);
- otherEventObservable.parallel(new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
- //only take while our stats tell us we should continue processing
- //potentially skip the first n if this is a resume operation
++ // only take while our stats tell us we should continue processing
++ // potentially skip the first n if this is a resume operation
+ otherEventObservable.takeWhile( new Func1<WriteEvent, Boolean>() {
@Override
- public Observable<WriteEvent> call(Observable<WriteEvent> entityWrapperObservable) {
- return entityWrapperObservable.doOnNext(doWork);
+ public Boolean call( final WriteEvent writeEvent ) {
+ return !statistics.shouldStopProcessingConnections();
}
- }, Schedulers.io()).toBlocking().last();
+ } ).skip( connectionNumSkip ).parallel( new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
+ @Override
+ public Observable<WriteEvent> call( Observable<WriteEvent> entityWrapperObservable ) {
+ return entityWrapperObservable.doOnNext( doWork );
+ }
+ }, Schedulers.io() ).toBlocking().last();
jp.close();
+ logger.debug("\n\nimportEntitiesFromFile(): Wrote others\n");
++
+ //flush the job statistics
+ statistics.complete();
-
- logger.debug("\n\nWrote others\n");
}
[11/16] incubator-usergrid git commit: Update by import test now
works.
Posted by sn...@apache.org.
Update by import test now works.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cdf63468
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cdf63468
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cdf63468
Branch: refs/heads/two-dot-o-import
Commit: cdf634685af004c2579212f8c7ea3866eba26045
Parents: 00e7194
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Feb 4 11:29:15 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Feb 4 11:29:15 2015 -0500
----------------------------------------------------------------------
.../management/importer/ImportServiceImpl.java | 9 +---
.../management/importer/ImportCollectionIT.java | 54 ++++++++------------
2 files changed, 23 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cdf63468/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index e122917..1ccc1db 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@ -797,7 +797,6 @@ public class ImportServiceImpl implements ImportService {
writeEvent.doWrite(em,fileImport);
}
};
-
// final AtomicLong entityCounter = new AtomicLong();
// final AtomicLong eventCounter = new AtomicLong();
@@ -805,7 +804,6 @@ public class ImportServiceImpl implements ImportService {
entityEventObservable.parallel(new Func1<Observable<WriteEvent>, Observable<WriteEvent>>() {
@Override
public Observable<WriteEvent> call(Observable<WriteEvent> entityWrapperObservable) {
-
// TODO: need to fixed so that number of entities created can be counted correctly and
// TODO: also update last updated UUID for fileImport which is a must for resume-ability
@@ -840,14 +838,12 @@ public class ImportServiceImpl implements ImportService {
// }
// }
// );
-
return entityWrapperObservable.doOnNext(doWork);
-
}
}, Schedulers.io()).toBlocking().last();
jp.close();
- logger.debug("\n\nWrote entities\n");
+ logger.debug("\n\nimportEntitiesFromFile(): Wrote entities\n");
// now do other stuff: connections and dictionaries
entitiesOnly = false;
@@ -862,13 +858,12 @@ public class ImportServiceImpl implements ImportService {
@Override
public Observable<WriteEvent> call(Observable<WriteEvent> entityWrapperObservable) {
return entityWrapperObservable.doOnNext(doWork);
-
}
}, Schedulers.io()).toBlocking().last();
jp.close();
- logger.debug("\n\nWrote others\n");
+ logger.debug("\n\nimportEntitiesFromFile(): Wrote others\n");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cdf63468/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index f103602..0cc6bd4 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -18,24 +18,19 @@
package org.apache.usergrid.management.importer;
import com.amazonaws.SDKGlobalConfiguration;
-import com.google.common.collect.BiMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Service;
import com.google.inject.Module;
import org.apache.commons.lang3.StringUtils;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
-import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.batch.service.JobSchedulerService;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.management.export.ExportService;
-import org.apache.usergrid.management.export.S3Export;
-import org.apache.usergrid.management.export.S3ExportImpl;
import org.apache.usergrid.persistence.*;
-import org.apache.usergrid.persistence.entities.JobData;
import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.persistence.index.query.Query.Level;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
@@ -43,7 +38,6 @@ import org.apache.usergrid.services.notifications.QueueListener;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.domain.Blob;
import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.netty.config.NettyPayloadModule;
@@ -53,11 +47,8 @@ import org.slf4j.LoggerFactory;
import java.util.*;
-import static org.hamcrest.core.Is.is;
-import static org.hamcrest.core.IsNot.not;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
//@Concurrent
@@ -126,32 +117,27 @@ public class ImportCollectionIT {
@Test
public void testExportImportCollection() throws Exception {
- // clear bucket at start of test
- deleteBucket();
-
- final EntityManager emApp1 = setup.getEmf().getEntityManager( applicationId );
- // create a collection of "thing" entities in the first application
+ // create a collection of "thing" entities in the first application, export to S3
+ final EntityManager emApp1 = setup.getEmf().getEntityManager( applicationId );
Map<UUID, Entity> thingsMap = new HashMap<>();
List<Entity> things = new ArrayList<>();
-
createTestEntities(emApp1, thingsMap, things, "thing");
- // export the "things" collection to a JSON file in an S3 bucket
-
+ deleteBucket();
exportCollection( emApp1, "things" );
- // create new second application
- final UUID appId2 = setup.getMgmtSvc().createApplication(
- organization.getUuid(), "secondapp").getId();
+ // create new second application, import the data from S3
- // import the data into the new application
+ final UUID appId2 = setup.getMgmtSvc().createApplication(
+ organization.getUuid(), "second").getId();
final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
importCollection( emApp2, "things" );
+
// make sure that it worked
try {
@@ -238,13 +224,13 @@ public class ImportCollectionIT {
// create new second application and import those things from S3
final UUID appId2 = setup.getMgmtSvc().createApplication(
- organization.getUuid(), "secondapp").getId();
+ organization.getUuid(), "second").getId();
final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
importCollection( emApp2, "things" );
- // update the things in the new application, export to S3
+ // update the things in the second application, export to S3
for ( UUID uuid : thingsMap.keySet() ) {
Entity entity = emApp2.get( uuid );
@@ -253,10 +239,10 @@ public class ImportCollectionIT {
}
deleteBucket();
- exportCollection( emApp1, "things" );
+ exportCollection( emApp2, "things" );
- // import the updated things into the first application, check that they've been updated
+ // import the updated things back into the first application, check that they've been updated
importCollection( emApp1, "things" );
@@ -324,7 +310,7 @@ public class ImportCollectionIT {
*/
private void importCollection(final EntityManager em, final String collectionName ) throws Exception {
- logger.debug("\n\nImport into new app {}\n", em.getApplication().getUuid() );
+ logger.debug("\n\nImport into new app {}\n", em.getApplication().getName() );
ImportService importService = setup.getImportService();
UUID importUUID = importService.schedule( new HashMap<String, Object>() {{
@@ -360,13 +346,16 @@ public class ImportCollectionIT {
private void exportCollection(
final EntityManager em, final String collectionName ) throws Exception {
- logger.debug("\n\nExporting {} collection\n", collectionName );
+ logger.debug("\n\nExporting {} collection from application {}\n",
+ collectionName, em.getApplication().getName() );
+
+ em.refreshIndex();
ExportService exportService = setup.getExportService();
UUID exportUUID = exportService.schedule( new HashMap<String, Object>() {{
put( "path", organization.getName() + em.getApplication().getName());
put( "organizationId", organization.getUuid());
- put( "applicationId", applicationId );
+ put( "applicationId", em.getApplication().getUuid() );
put( "collectionName", collectionName);
put( "properties", new HashMap<String, Object>() {{
put( "storage_provider", "s3" );
@@ -393,7 +382,8 @@ public class ImportCollectionIT {
private void createTestEntities( final EntityManager em,
Map<UUID, Entity> thingsMap, List<Entity> things, final String type) throws Exception {
- logger.debug("\n\nCreating {} collection\n", type);
+ logger.debug("\n\nCreating new {} collection in application {}\n",
+ type, em.getApplication().getName() );
for ( int i = 0; i < 10; i++ ) {
final int count = i;
@@ -406,8 +396,6 @@ public class ImportCollectionIT {
things.add( e );
}
- logger.debug("\n\nCreate Connections\n");
-
// first two things are related to each other
em.createConnection( new SimpleEntityRef( "thing", things.get(0).getUuid()),
"related", new SimpleEntityRef( "thing", things.get(1).getUuid()));
[02/16] incubator-usergrid git commit: remove lexer
Posted by sn...@apache.org.
remove lexer
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/df29a99c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/df29a99c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/df29a99c
Branch: refs/heads/two-dot-o-import
Commit: df29a99c4218a39269151a8773c0b0e051507650
Parents: 0fc961e
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Jan 27 09:11:04 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Jan 27 09:11:04 2015 -0700
----------------------------------------------------------------------
.gitignore | 6 ------
1 file changed, 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df29a99c/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 522f674..6f8e854 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,6 @@
.DS_Store
.AppleDouble
.LSOverride
-stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
-stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
target
@@ -41,8 +39,6 @@ stack/corepersistence/graph/nbactions.xml
stack/corepersistence/model/nbactions.xml
stack/corepersistence/nbactions.xml
stack/corepersistence/queryindex/nbactions.xml
-stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
-stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
/stack/corepersistence/perftest2/src/main/resources/finefoods.txt
/stack/corepersistence/perftest1/src/main/resources/finefoods.txt
/stack/corepersistence/perftest2/nbactions.xml
@@ -80,8 +76,6 @@ portal/js/templates.js
/release/target/
/portal/seleniumLog.txt
-/stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
-/stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
!/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
*.iml
sdks/dotnet/samples/notifications/packages/*
[05/16] incubator-usergrid git commit: fixed double slash bug in data
explorer in admin portal
Posted by sn...@apache.org.
fixed double slash bug in data explorer in admin portal
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/373248be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/373248be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/373248be
Branch: refs/heads/two-dot-o-import
Commit: 373248bed240df43e3e769b60a425dbbc9f61c07
Parents: 6dc62ed
Author: Rod Simpson <ro...@apigee.com>
Authored: Tue Jan 27 13:24:35 2015 -0600
Committer: Rod Simpson <ro...@apigee.com>
Committed: Tue Jan 27 13:24:35 2015 -0600
----------------------------------------------------------------------
portal/js/global/ug-service.js | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/373248be/portal/js/global/ug-service.js
----------------------------------------------------------------------
diff --git a/portal/js/global/ug-service.js b/portal/js/global/ug-service.js
index 3401060..c4e0d53 100644
--- a/portal/js/global/ug-service.js
+++ b/portal/js/global/ug-service.js
@@ -448,8 +448,9 @@ AppServices.Services.factory('ug', function(configuration, $rootScope, utility,
} else {
var queryPath = data.path;
- self.getCollection('query', queryPath, null,
- 'order by modified DESC', null);
+ //remove preceeding slash
+ queryPath = queryPath.replace(/^\//, '');
+ self.getCollection('query', queryPath, null, 'order by modified DESC', null);
}
});
[04/16] incubator-usergrid git commit: remove lexer
Posted by sn...@apache.org.
remove lexer
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6da8119f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6da8119f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6da8119f
Branch: refs/heads/two-dot-o-import
Commit: 6da8119f8fef8d3b42c90bbfb9053baa3b60d7c4
Parents: eafdf63
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Jan 27 09:19:55 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Jan 27 09:19:55 2015 -0700
----------------------------------------------------------------------
.gitignore | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6da8119f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 522f674..b516ec8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,6 @@
.DS_Store
.AppleDouble
.LSOverride
-stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterParser.java
-stack/core/src/main/java/org/apache/usergrid/persistence/query/tree/QueryFilterLexer.java
-
target
# Icon must ends with two \r.
[09/16] incubator-usergrid git commit: first pass of new update by
import test.
Posted by sn...@apache.org.
first pass of new update by import test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3cdf50d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3cdf50d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3cdf50d3
Branch: refs/heads/two-dot-o-import
Commit: 3cdf50d3bdaa55c69b3d15dc03046750f3c52caa
Parents: 68894d9
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Feb 4 10:28:19 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Feb 4 10:28:19 2015 -0500
----------------------------------------------------------------------
.../management/importer/ImportCollectionIT.java | 212 +++++++++++++++----
1 file changed, 169 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3cdf50d3/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index 3940e3b..f103602 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -126,28 +126,31 @@ public class ImportCollectionIT {
@Test
public void testExportImportCollection() throws Exception {
+ // clear bucket at start of test
+ deleteBucket();
+
final EntityManager emApp1 = setup.getEmf().getEntityManager( applicationId );
- // create a collection of "thing" entities in the default test application
+ // create a collection of "thing" entities in the first application
Map<UUID, Entity> thingsMap = new HashMap<>();
List<Entity> things = new ArrayList<>();
- createTestEntities(emApp1, thingsMap, things);
+ createTestEntities(emApp1, thingsMap, things, "thing");
// export the "things" collection to a JSON file in an S3 bucket
- exportCollection(emApp1);
+ exportCollection( emApp1, "things" );
- // create new application
+ // create new second application
final UUID appId2 = setup.getMgmtSvc().createApplication(
- organization.getUuid(), "noobapp" ).getId();
+ organization.getUuid(), "secondapp").getId();
// import the data into the new application
final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
- importCollection(appId2, emApp2);
+ importCollection( emApp2, "things" );
// make sure that it worked
@@ -159,6 +162,7 @@ public class ImportCollectionIT {
assertTrue( !importedThings.isEmpty() );
// two things have connections
+
int conCount = 0;
for ( Entity e : importedThings ) {
Results r = emApp2.getConnectedEntities( e, "related", null, Level.IDS);
@@ -167,29 +171,33 @@ public class ImportCollectionIT {
}
assertEquals( 2, conCount );
- logger.debug("\n\nCheck dictionary\n");
+ logger.debug("\n\nCheck dictionaries\n");
- // check if dictionary is created
- EntityRef er;
- Map<Object, Object> dictionaries1, dictionaries2;
+ // first two items have things in dictionary
- for (int i = 0; i < 3; i++) {
+ EntityRef entity0 = importedThings.get(0);
+ Map connected0 = emApp2.getDictionaryAsMap(entity0, "connected_types");
+ Map connecting0 = emApp2.getDictionaryAsMap(entity0, "connected_types");
+ Assert.assertEquals( 1, connected0.size() );
+ Assert.assertEquals( 1, connecting0.size() );
- er = importedThings.get(i);
- dictionaries1 = emApp2.getDictionaryAsMap(er, "connected_types");
- dictionaries2 = emApp2.getDictionaryAsMap(er, "connecting_types");
+ EntityRef entity1 = importedThings.get(1);
+ Map connected1 = emApp2.getDictionaryAsMap(entity1, "connected_types");
+ Map connecting1 = emApp2.getDictionaryAsMap(entity1, "connected_types");
+ Assert.assertEquals( 1, connected1.size() );
+ Assert.assertEquals( 1, connecting1.size() );
- if (i == 2) {
- //for entity 2, these should be empty
- assertThat(dictionaries1.size(), is(0));
- assertThat(dictionaries2.size(), is(0));
- } else {
- assertThat(dictionaries1.size(), is(not(0)));
- assertThat(dictionaries2.size(), is(not(0)));
- }
- }
+ // rest do not
+
+ EntityRef entity2 = importedThings.get(2);
+ Map connected2 = emApp2.getDictionaryAsMap(entity2, "connected_types");
+ Map connecting2 = emApp2.getDictionaryAsMap(entity2, "connected_types");
+ Assert.assertEquals( 0, connected2.size() );
+ Assert.assertEquals( 0, connecting2.size() );
// if entities are deleted from app1, they still exist in app2
+
+ logger.debug("\n\nCheck dictionary\n");
for ( Entity importedThing : importedThings ) {
emApp1.delete( importedThing );
}
@@ -206,16 +214,124 @@ public class ImportCollectionIT {
}
}
- private void importCollection(final UUID appId2, final EntityManager em2) throws Exception {
- logger.debug("\n\nImport into new app {}\n", appId2 );
+ /**
+ * Test that an existing collection of entities can be updated
+ * by doing an import of entities identified by UUIDs.
+ */
+ @Test
+ public void testUpdateByImport() throws Exception {
+
+
+ // create collection of things in first application, export them to S3
+
+ final EntityManager emApp1 = setup.getEmf().getEntityManager( applicationId );
+
+ Map<UUID, Entity> thingsMap = new HashMap<>();
+ List<Entity> things = new ArrayList<>();
+ createTestEntities(emApp1, thingsMap, things, "thing");
+
+ deleteBucket();
+ exportCollection( emApp1, "things" );
+
+
+ // create new second application and import those things from S3
+
+ final UUID appId2 = setup.getMgmtSvc().createApplication(
+ organization.getUuid(), "secondapp").getId();
+
+ final EntityManager emApp2 = setup.getEmf().getEntityManager(appId2);
+ importCollection( emApp2, "things" );
+
+
+ // update the things in the new application, export to S3
+
+ for ( UUID uuid : thingsMap.keySet() ) {
+ Entity entity = emApp2.get( uuid );
+ entity.setProperty("fuel_source", "Hydrogen");
+ emApp2.update( entity );
+ }
+
+ deleteBucket();
+ exportCollection( emApp1, "things" );
+
+
+ // import the updated things into the first application, check that they've been updated
+
+ importCollection( emApp1, "things" );
+
+ for ( UUID uuid : thingsMap.keySet() ) {
+ Entity entity = emApp1.get( uuid );
+ Assert.assertEquals("Hydrogen", entity.getProperty("fuel_source"));
+ }
+ }
+
+
+ /**
+ * Test that an existing collection of entities can be updated
+ * by doing an import of entities identified by UUIDs.
+ */
+ @Test
+ public void testCreateByImport() {
+
+ // import from a JSON file stored locally on disk, with no UUID or type info
+
+ // check that entities were created
+
+ }
+
+
+ /**
+ * Test that the types of incoming entities is ignored.
+ */
+ @Test
+ public void testImportWithWrongTypes() {
+
+ // import from a JSON file stored locally on disk, with incorrect type for collection
+
+ // check that entities were created with collection's type
+
+ }
+
+
+ /**
+ * Test that importing bad JSON will result in an informative error message.
+ */
+ @Test
+ public void testImportBadJson() {
+
+ // import from a bad JSON file
+
+ // check that error message indicates JSON parsing error
+ }
+
+
+ /**
+ * Simple import test but with multiple files.
+ */
+ @Test
+ public void testImportWithMultipleFiles() {
+
+ // create collection of things in first app, export them to S3
+
+ // create collection of things in second app, export them to S3
+ }
+
+
+ /**
+ * Call importService to import files from the configured S3 bucket.
+ * @param collectionName Name of collection into which Entities will be imported.
+ */
+ private void importCollection(final EntityManager em, final String collectionName ) throws Exception {
+
+ logger.debug("\n\nImport into new app {}\n", em.getApplication().getUuid() );
ImportService importService = setup.getImportService();
UUID importUUID = importService.schedule( new HashMap<String, Object>() {{
- put( "path", organization.getName() + em2.getApplication().getName());
+ put( "path", organization.getName() + em.getApplication().getName());
put( "organizationId", organization.getUuid());
- put( "applicationId", appId2 );
- put( "collectionName", "things");
+ put( "applicationId", em.getApplication().getUuid() );
+ put( "collectionName", collectionName);
put( "properties", new HashMap<String, Object>() {{
put( "storage_provider", "s3" );
put( "storage_info", new HashMap<String, Object>() {{
@@ -235,19 +351,23 @@ public class ImportCollectionIT {
Thread.sleep(100);
}
- em2.refreshIndex();
+ em.refreshIndex();
}
- private void exportCollection(final EntityManager em) throws Exception {
+ /**
+ * Call exportService to export the named collection to the configured S3 bucket.
+ */
+ private void exportCollection(
+ final EntityManager em, final String collectionName ) throws Exception {
- logger.debug("\n\nExport\n");
+ logger.debug("\n\nExporting {} collection\n", collectionName );
ExportService exportService = setup.getExportService();
UUID exportUUID = exportService.schedule( new HashMap<String, Object>() {{
put( "path", organization.getName() + em.getApplication().getName());
put( "organizationId", organization.getUuid());
put( "applicationId", applicationId );
- put( "collectionName", "things");
+ put( "collectionName", collectionName);
put( "properties", new HashMap<String, Object>() {{
put( "storage_provider", "s3" );
put( "storage_info", new HashMap<String, Object>() {{
@@ -266,16 +386,21 @@ public class ImportCollectionIT {
}
}
- private void createTestEntities(
- EntityManager em, Map<UUID, Entity> thingsMap, List<Entity> things) throws Exception {
+ /**
+ * Create test entities of a specified type.
+ * First two entities are connected.
+ */
+ private void createTestEntities( final EntityManager em,
+ Map<UUID, Entity> thingsMap, List<Entity> things, final String type) throws Exception {
- logger.debug("\n\nCreate things collection\n");
+ logger.debug("\n\nCreating {} collection\n", type);
for ( int i = 0; i < 10; i++ ) {
final int count = i;
- Entity e = em.create( "thing", new HashMap<String, Object>() {{
- put("name", "thing" + count);
- put("index", count);
+ Entity e = em.create( type, new HashMap<String, Object>() {{
+ put("name", em.getApplication().getName() + "-" + type + "-" + count);
+ put("originalAppId", em.getApplication().getUuid());
+ put("originalAppName", em.getApplication().getName());
}});
thingsMap.put(e.getUuid(), e);
things.add( e );
@@ -300,7 +425,9 @@ public class ImportCollectionIT {
}
}
- // delete the s3 bucket which was created for testing
+ /**
+ * Delete the configured s3 bucket.
+ */
public void deleteBucket() {
logger.debug("\n\nDelete bucket\n");
@@ -313,17 +440,16 @@ public class ImportCollectionIT {
overrides.setProperty( "s3" + ".identity", accessId );
overrides.setProperty( "s3" + ".credential", secretKey );
- Blob bo = null;
- BlobStore blobStore = null;
final Iterable<? extends Module> MODULES = ImmutableSet
- .of(new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
+ .of(new JavaUrlHttpCommandExecutorServiceModule(),
+ new Log4JLoggingModule(),
new NettyPayloadModule());
BlobStoreContext context =
ContextBuilder.newBuilder("s3").credentials( accessId, secretKey ).modules( MODULES )
.overrides( overrides ).buildView( BlobStoreContext.class );
- blobStore = context.getBlobStore();
+ BlobStore blobStore = context.getBlobStore();
blobStore.deleteContainer( bucketName );
}
[07/16] incubator-usergrid git commit: fix authentication workflow
Posted by sn...@apache.org.
fix authentication workflow
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bfdacb34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bfdacb34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bfdacb34
Branch: refs/heads/two-dot-o-import
Commit: bfdacb34ebdf918fcf92307c14808cadd5bdd7fd
Parents: 6ba340a
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Jan 27 19:31:58 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Jan 27 19:31:58 2015 -0700
----------------------------------------------------------------------
.../Usergrid.Notifications/Client/PushClient.cs | 12 +++++++-----
.../Usergrid.Notifications/Client/Usergrid.cs | 10 ++++++----
.../Usergrid.Notifications/MainPage.xaml.cs | 15 ++++++---------
3 files changed, 19 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfdacb34/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
index 8668626..fb8860c 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
@@ -35,12 +35,14 @@ namespace Usergrid.Notifications.Client
private IUsergridHttpClient usergrid;
private ApplicationDataContainer settings;
private PushNotificationChannel channel;
+ private string userId;
- public PushClient(IUsergridHttpClient usergrid, string notifier)
+ public PushClient(IUsergridHttpClient usergrid,string userId, string notifier)
{
this.usergrid = usergrid;
this.settings = ApplicationData.Current.LocalSettings;
this.Notifier = notifier;
+ this.userId = userId;
this.init().ContinueWith(t =>
{
LastException = t.Exception;
@@ -62,7 +64,7 @@ namespace Usergrid.Notifications.Client
payloads.Add(Notifier, payload);
jsonObject.Add("payloads", payloads);
jsonObject.Add("debug", true);
- var jsonResponse = await usergrid.SendAsync(HttpMethod.Post, String.Format("devices/{0}/notifications", this.DeviceId), jsonObject);
+ var jsonResponse = await usergrid.SendAsync(HttpMethod.Post, String.Format("users/{1}/devices/{0}/notifications", this.DeviceId,userId), jsonObject);
return jsonResponse.StatusIsOk;
}
@@ -79,7 +81,7 @@ namespace Usergrid.Notifications.Client
payloads.Add(Notifier, payload);
jsonObject.Add("payloads", payloads);
jsonObject.Add("debug", true);
- var jsonResponse = await usergrid.SendAsync(HttpMethod.Post, String.Format("devices/{0}/notifications", this.DeviceId), jsonObject);
+ var jsonResponse = await usergrid.SendAsync(HttpMethod.Post, String.Format("users/{1}/devices/{0}/notifications", this.DeviceId,userId), jsonObject);
return jsonResponse.StatusIsOk;
}
@@ -114,7 +116,7 @@ namespace Usergrid.Notifications.Client
private async Task<JToken> GetDevice(Guid deviceId)
{
- var jsonResponse = await usergrid.SendAsync(HttpMethod.Get, "devices/" + deviceId, null);
+ var jsonResponse = await usergrid.SendAsync(HttpMethod.Get, "users/"+userId+"/devices/" + deviceId, null);
if (jsonResponse.StatusIsOk)
{
@@ -130,7 +132,7 @@ namespace Usergrid.Notifications.Client
obj.Add(Notifier + ".notifier.id", new JValue(channel.Uri));
var jsonResponse = await usergrid.SendAsync(
(isNew ? HttpMethod.Post : HttpMethod.Put),
- "devices/" + (isNew ? "" : this.DeviceId.ToString()),
+ "users/"+userId+"/devices/" + (isNew ? "" : this.DeviceId.ToString()),
obj
);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfdacb34/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/Usergrid.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/Usergrid.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/Usergrid.cs
index cd0330d..977eb8b 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/Usergrid.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/Usergrid.cs
@@ -48,13 +48,15 @@ namespace Usergrid.Notifications.Client
/// <param name="org"></param>
/// <param name="app"></param>
/// <param name="channel"></param>
- internal Usergrid(string serverUrl, string org, string app, string notifier)
+ internal Usergrid(string serverUrl, string org, string app, string userId, string password,string notifier)
{
string serverUrlWithSlash = serverUrl.EndsWith("/", StringComparison.CurrentCulture) ? serverUrl : serverUrl + "/";
this.appUrl = String.Format("{0}{1}/{2}/", serverUrlWithSlash, org, app);
this.managementUrl = serverUrlWithSlash + "management/";
this.client = new HttpClient();
- this.push = new PushClient(this,notifier);
+ Authenticate(userId, password, false).ContinueWith(task => {
+ this.push = new PushClient(this, userId, notifier);
+ });
}
public async Task Authenticate(string user, string password, bool isManagement)
@@ -69,7 +71,7 @@ namespace Usergrid.Notifications.Client
if (response.StatusIsOk)
{
this.token = response.GetValue("access_token").Value<String>();
- client.DefaultRequestHeaders.Add("X-Authorization", token);
+ client.DefaultRequestHeaders.Add("Authorization", "Bearer "+ token);
}
else
{
@@ -89,7 +91,7 @@ namespace Usergrid.Notifications.Client
public async Task<EntityResponse> SendAsync(HttpMethod method, string url, object obj, bool useManagementUrl)
{
- HttpRequestMessage message = new HttpRequestMessage(HttpMethod.Post, (useManagementUrl ? this.managementUrl : this.appUrl) + url);
+ HttpRequestMessage message = new HttpRequestMessage(method, (useManagementUrl ? this.managementUrl : this.appUrl) + url);
if (obj != null)
{
message.Content = getJsonBody(obj);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bfdacb34/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
index 7e03c79..6d4c4ec 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
@@ -44,27 +44,24 @@ namespace Usergrid.Notifications
{
this.InitializeComponent();
//TODO: change me to your server
- serverUrl = "http://10.0.1.20:8080";
+ serverUrl = "https://apigee-internal-prod.mars.apigee.net";
//TODO: change me to your org
- org = "mobile";
+ org = "ugvalidate";
//TODO: change me to your app
app = "sandbox";
//TODO: change me to your notifier name
notifier = "winphone";
//TODO: change me to your user
- user = "superuser";
+ user = "mobileuser";
//TODO: change me to your password
- password = "test";
+ password = "******";
this.NavigationCacheMode = NavigationCacheMode.Required;
- this.setup().ContinueWith(t => {
- LastException = t.Exception;
- });
+ usergrid = new Client.Usergrid(serverUrl, org, app, user, password, notifier);
+
}
private async Task setup()
{
- usergrid = new Client.Usergrid(serverUrl, org, app, notifier);
- await usergrid.Authenticate(user, password, true);
}
/// <summary>
[16/16] incubator-usergrid git commit: Merge branch 'two-dot-o' into
two-dot-o-import
Posted by sn...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-import
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fd4d47bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fd4d47bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fd4d47bb
Branch: refs/heads/two-dot-o-import
Commit: fd4d47bb3fa0b84100a2ad89983bfdcd12e66d92
Parents: 54d31cd 198f479
Author: Dave Johnson <dm...@apigee.com>
Authored: Thu Feb 5 09:45:14 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Thu Feb 5 09:45:14 2015 -0500
----------------------------------------------------------------------
.gitignore | 3 ---
portal/js/global/ug-service.js | 5 +++--
.../Usergrid.Notifications/Client/PushClient.cs | 12 +++++++-----
.../Usergrid.Notifications/Client/Usergrid.cs | 10 ++++++----
.../Usergrid.Notifications/MainPage.xaml.cs | 15 ++++++---------
5 files changed, 22 insertions(+), 23 deletions(-)
----------------------------------------------------------------------