You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by md...@apache.org on 2017/08/28 23:23:47 UTC
[01/15] usergrid git commit: Add new parameter that can be used in
the JSP viewables.
Repository: usergrid
Updated Branches:
refs/heads/master 36e28a870 -> 8e9eff8d9
Add new parameter that can be used in the JSP viewables.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/068263ed
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/068263ed
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/068263ed
Branch: refs/heads/master
Commit: 068263ed437883edeba1d23a024f62bb6e7b0700
Parents: a6fee78
Author: Michael Russo <ru...@google.com>
Authored: Wed May 24 21:45:04 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 09:41:40 2017 -0700
----------------------------------------------------------------------
stack/config/src/main/resources/usergrid-default.properties | 1 +
.../apache/usergrid/rest/management/users/UserResource.java | 7 +++++++
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/068263ed/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index d505a96..77b047e 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -786,6 +786,7 @@ usergrid.api.url.base=http://localhost:8080/ROOT
# instead, use character class ([.] instead of backslash-period)
usergrid.org.config.property.regex=usergrid[.]view[.].*
+usergrid.viewable.loginEndpoint=http://localhost:8080
########################### Usergrid Email Templates ########################
http://git-wip-us.apache.org/repos/asf/usergrid/blob/068263ed/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index 4cbe9b2..b747aa4 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -66,6 +66,9 @@ public class UserResource extends AbstractContextResource {
String token = null;
+ String loginEndpoint;
+
+
public UserResource() {
}
@@ -296,6 +299,7 @@ public class UserResource extends AbstractContextResource {
if ( ( password1 != null ) && password1.equals( password2 ) ) {
management.setAdminUserPassword( user.getUuid(), password1 );
management.revokeAccessTokenForAdminUser( user.getUuid(), token );
+ loginEndpoint = properties.getProperty("usergrid.viewable.loginEndpoint");
return handleViewable( "resetpw_set_success", this, organizationId );
}
else {
@@ -342,6 +346,9 @@ public class UserResource extends AbstractContextResource {
return errorMsg;
}
+ public String getLoginEndpoint() {
+ return loginEndpoint;
+ }
public String getToken() {
return token;
[13/15] usergrid git commit: 1. Add missing start for delete queue
workers. 2. Add dropping of application local keyspace for testing. 3.
Testing changes.
Posted by md...@apache.org.
1. Add missing start for delete queue workers.
2. Add dropping of application local keyspace for testing.
3. Testing changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3f7afcd8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3f7afcd8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3f7afcd8
Branch: refs/heads/master
Commit: 3f7afcd8836ed9b589b69523d8bcc507a739d912
Parents: c201f1f
Author: Mike Dunker <md...@google.com>
Authored: Mon Aug 21 17:01:48 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Aug 21 17:01:48 2017 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 3 +--
.../asyncevents/AsyncEventServiceImpl.java | 4 +++
.../index/CollectionSettingsImpl.java | 1 -
.../usergrid/persistence/EntityManager.java | 2 +-
.../persistence/entities/Application.java | 26 ++++++++++++++++++++
.../usergrid/persistence/CoreSchemaManager.java | 16 ++++++++----
.../usergrid/persistence/RebuildIndexTest.java | 24 +++++++++++++-----
.../resources/usergrid-custom-test.properties | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
.../AbstractServiceNotificationIT.java | 4 +--
.../gcm/NotificationsServiceIT.java | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
12 files changed, 67 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index cdb4fc7..68c4ef0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2471,10 +2471,9 @@ public class CpEntityManager implements EntityManager {
final Entity entity;
- //this is the fall back, why isn't this writt
if ( entityType == null ) {
return null;
-// throw new EntityNotFoundException( String.format( "Counld not find type for uuid {}", uuid ) );
+// throw new EntityNotFoundException( String.format( "Could not find type for uuid {}", uuid ) );
}
entity = get( new SimpleEntityRef( entityType, uuid ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 79a80c0..428772f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -875,6 +875,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
startWorker(AsyncEventQueueType.UTILITY);
}
+ for (int i = 0; i < deleteCount; i++) {
+ startWorker(AsyncEventQueueType.DELETE);
+ }
+
if( indexQueue instanceof SNSQueueManagerImpl) {
logger.info("Queue manager implementation supports dead letters, start dead letter queue workers.");
for (int i = 0; i < indexDeadCount; i++) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
index 921777a..76f0b4b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.corepersistence.index;
import com.google.common.base.Optional;
import com.google.inject.Inject;
-import com.google.inject.Singleton;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.utils.JsonUtils;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index ae4623d..c9752c3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -743,7 +743,7 @@ public interface EntityManager {
/**
* Add a new index to the application for scale
- * @param suffix unique indentifier for additional index
+ * @param newIndexName unique identifier for additional index
* @param shards number of shards
* @param replicas number of replicas
* @param writeConsistency only "one, quorum, or all"
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
index 790b4d9..0a4360f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
@@ -48,6 +48,32 @@ public class Application extends TypedEntity implements Serializable {
public static final String COLLECTION_ACTIVITIES = "activities";
+ public static final String COLLECTION_EVENTS = "events";
+
+ public static final String COLLECTION_FOLDERS = "folders";
+
+ public static final String COLLECTION_DEVICES = "devices";
+
+ public static final String COLLECTION_NOTIFICATIONS = "notifications";
+
+ public static final String COLLECTION_ROLES = "roles";
+
+ public static boolean isCustomCollectionName(String collectionName) {
+ switch (collectionName.toLowerCase()) {
+ case COLLECTION_USERS:
+ case COLLECTION_GROUPS:
+ case COLLECTION_ASSETS:
+ case COLLECTION_ACTIVITIES:
+ case COLLECTION_EVENTS:
+ case COLLECTION_FOLDERS:
+ case COLLECTION_DEVICES:
+ case COLLECTION_NOTIFICATIONS:
+ case COLLECTION_ROLES:
+ return false;
+ }
+ return true;
+ }
+
@EntityProperty(indexed = true, fulltextIndexed = false, required = true, mutable = false, aliasProperty = true,
basic = true)
protected String name;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
index c064751..1d51c25 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.persistence;
import org.apache.usergrid.locking.LockManager;
+import org.apache.usergrid.persistence.core.CassandraConfig;
import org.apache.usergrid.persistence.core.CassandraFig;
import org.apache.usergrid.persistence.core.datastax.CQLUtils;
import org.apache.usergrid.persistence.core.datastax.DataStaxCluster;
@@ -37,14 +38,14 @@ public class CoreSchemaManager implements SchemaManager {
private static final Logger logger = LoggerFactory.getLogger( CoreSchemaManager.class );
private final Setup setup;
- private final CassandraFig cassandraFig;
+ private final CassandraConfig cassandraConfig;
private final LockManager lockManager;
private final DataStaxCluster dataStaxCluster;
public CoreSchemaManager( final Setup setup, Injector injector ) {
this.setup = setup;
- this.cassandraFig = injector.getInstance( CassandraFig.class );
+ this.cassandraConfig = injector.getInstance( CassandraConfig.class );
this.lockManager = injector.getInstance( LockManager.class );
this.dataStaxCluster = injector.getInstance( DataStaxCluster.class );
}
@@ -80,16 +81,21 @@ public class CoreSchemaManager implements SchemaManager {
@Override
public void destroy() {
- logger.info( "dropping keyspaces" );
try {
+ logger.info( "dropping application keyspace" );
dataStaxCluster.getClusterSession()
- .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraFig.getApplicationKeyspace()));
+ .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationKeyspace()));
+ dataStaxCluster.waitForSchemaAgreement();
+
+ logger.info( "dropping application local keyspace" );
+ dataStaxCluster.getClusterSession()
+ .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationLocalKeyspace()));
dataStaxCluster.waitForSchemaAgreement();
dataStaxCluster.getClusterSession().close(); // close session so it's meta will get refreshed
}
catch ( Exception e ) {
- logger.error("Error dropping application keyspace: {} error: {}", cassandraFig.getApplicationKeyspace(), e);
+ logger.error("Error dropping application keyspaces: {} error: {}", cassandraConfig.getApplicationKeyspace(), e);
}
logger.info( "keyspaces dropped" );
logger.info( "dropping indices" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index d2bff37..a1ff84b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -74,7 +74,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
}
- @Test( timeout = 120000 )
+ @Test( timeout = 240000 )
public void rebuildOneCollectionIndex() throws Exception {
logger.info( "Started rebuildOneCollectionIndex()" );
@@ -163,7 +163,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
waitForRebuild( status, reIndexService );
- app.waitForQueueDrainAndRefreshIndex(5000);
+ app.waitForQueueDrainAndRefreshIndex(15000);
// ----------------- test that we can read the catherder collection and not the catshepard
@@ -278,7 +278,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
assertNotNull( status.getJobId(), "JobId is present" );
- logger.info( "Rebuilt index" );
+ logger.info( "Rebuilt index, jobID={}", status.getJobId());
waitForRebuild( status, reIndexService );
@@ -511,14 +511,26 @@ public class RebuildIndexTest extends AbstractCoreIT {
* Wait for the rebuild to occur
*/
private void waitForRebuild( final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService )
- throws InterruptedException {
+ throws InterruptedException, IllegalArgumentException {
+ if (status != null) {
+ logger.info("waitForRebuild: jobID={}", status.getJobId());
+ } else {
+ logger.info("waitForRebuild: error, status = null");
+ throw new IllegalArgumentException("reindexStatus = null");
+ }
while ( true ) {
try {
final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus( status.getJobId() );
- if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
- break;
+ if (updatedStatus == null) {
+ logger.info("waitForRebuild: updated status is null");
+ } else {
+ logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed());
+
+ if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
+ break;
+ }
}
}
catch ( IllegalArgumentException iae ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index df253f0..24f3046 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -29,7 +29,7 @@ elasticsearch.management_number_replicas=0
elasticsearch.managment_index=usergrid_core_management
#cassandra.keyspace.application=core_tests_schema
elasticsearch.queue_impl.resolution=true
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
elasticsearch.buffer_timeout=1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 4b76ca4..ee7b039 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -68,7 +68,7 @@ usergrid.cluster.port=2555
collection.uniquevalues.actors=300
collection.uniquevalues.authoritative.region=us-east
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
# Queueing Test Settings
# Reduce the long polling time for the tests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
index c035192..9b3dfcd 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
@@ -49,9 +49,9 @@ public abstract class AbstractServiceNotificationIT extends AbstractServiceIT {
protected Notification notificationWaitForComplete(Notification notification)
throws Exception {
- long timeout = System.currentTimeMillis() + 60000;
+ long timeout = System.currentTimeMillis() + 120000;
while (System.currentTimeMillis() < timeout) {
- app.waitForQueueDrainAndRefreshIndex(200);
+ app.waitForQueueDrainAndRefreshIndex(1000);
notification = app.getEntityManager().get(notification.getUuid(), Notification.class);
if (notification.getFinished() != null) {
return notification;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 8360009..ecc5443 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -632,7 +632,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
// receipts are created and queried, wait a bit longer for this to happen as indexing
- app.waitForQueueDrainAndRefreshIndex(500);
+ app.waitForQueueDrainAndRefreshIndex(5000);
// get the receipts entity IDs
List<EntityRef> receipts = getNotificationReceipts(notification);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties
index bcc8b8e..4178a07 100644
--- a/stack/services/src/test/resources/usergrid-custom-test.properties
+++ b/stack/services/src/test/resources/usergrid-custom-test.properties
@@ -35,7 +35,7 @@ elasticsearch.managment_index=usergrid_services_management
elasticsearch.buffer_timeout=1
elasticsearch.queue_impl.resolution=true
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
# Queueing Test Settings
queue.long.polling.time.millis=50
[08/15] usergrid git commit: Basic code improvements 1. Increase some
testing timeouts. 2. improve some logging statements. 3. Fix duplicate
dependency in cored-persistence pom.xml 4. Accept SNS or DISTRIBUTED_SNS as
SNS queue types. 5. Drop missed test
Posted by md...@apache.org.
Basic code improvements
1. Increase some testing timeouts.
2. improve some logging statements.
3. Fix duplicate dependency in cored-persistence pom.xml
4. Accept SNS or DISTRIBUTED_SNS as SNS queue types.
5. Drop missed test keyspaces for qakka tests.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/400365c3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/400365c3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/400365c3
Branch: refs/heads/master
Commit: 400365c36eb1a46ba76ca3b2cf8f2db6670227b1
Parents: 25f1b0e
Author: Mike Dunker <md...@google.com>
Authored: Thu Aug 17 08:59:18 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 13:37:06 2017 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
.../actorsystem/ActorSystemManagerImpl.java | 3 ++-
.../persistence/index/impl/EsEntityIndexImpl.java | 2 +-
stack/corepersistence/queue/pom.xml | 18 ++++++------------
.../persistence/queue/guice/QueueModule.java | 2 +-
.../persistence/qakka/KeyspaceDropper.java | 6 +++++-
.../qakka/core/QueueMessageManagerTest.java | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
9 files changed, 19 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 77b047e..bf1f5e7 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -112,7 +112,7 @@ cassandra.embedded=false
# Read timeout for an individual request (in millseconds)
#
-#cassandra.timeout=5000
+#cassandra.timeout=20000
# Set the credentials used for Cassandra, if any.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index e32b146..df253f0 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -16,7 +16,7 @@
# with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml)
#cassandra.connections=30
cassandra.timeout.pool=20000
-cassandra.timeout=20000
+cassandra.timeout=25000
#Not a good number for real systems. Write shards should be 2x cluster size from our tests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index bfa3abe..ea9ada8 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -377,6 +377,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
Iterator<Class> messageTypes = routerProducer.getMessageTypes().iterator();
while ( messageTypes.hasNext() ) {
Class messageType = messageTypes.next();
+ logger.info("createClusterSystem: routerProducer {}: message type={}", routerProducer.getRouterPath(), messageType.getName());
routersByMessageType.put( messageType, routerProducer.getRouterPath() );
}
}
@@ -467,7 +468,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
if (started) {
logger.info( "ClientActor [{}] has started", ra.path() );
} else {
- throw new RuntimeException( "ClientActor ["+ra.path()+"] did not start in time" );
+ throw new RuntimeException( "ClientActor ["+ra.path()+"] did not start in time, validate that akka seeds are configured properly" );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index a35921c..cb695d5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -502,7 +502,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
searchResponse = srb.execute().actionGet();
}
catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t.getMessage() );
+ logger.error( "Unable to communicate with Elasticsearch: {}", t.getMessage() );
failureMonitor.fail( "Unable to execute batch", t );
throw t;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queue/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/pom.xml b/stack/corepersistence/queue/pom.xml
index 153ed4b..005ce0c 100644
--- a/stack/corepersistence/queue/pom.xml
+++ b/stack/corepersistence/queue/pom.xml
@@ -93,6 +93,12 @@
<groupId>org.apache.usergrid</groupId>
<artifactId>common</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>cassandra-all</artifactId>
+ <groupId>org.apache.cassandra</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
@@ -152,18 +158,6 @@
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>common</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>cassandra-all</artifactId>
- <groupId>org.apache.cassandra</groupId>
- </exclusion>
- </exclusions>
- </dependency>
-
<!--
<dependency>
<groupId>com.datastax.cassandra</groupId>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
index a485f55..a6e2451 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
@@ -43,7 +43,7 @@ public class QueueModule extends AbstractModule {
public QueueModule( String queueManagerType ) {
logger.info("QueueManagerType={}", queueManagerType);
- if ( "DISTRIBUTED_SNS".equals( queueManagerType ) ) {
+ if ( "DISTRIBUTED_SNS".equals( queueManagerType ) || "SNS".equals(queueManagerType)) {
this.implementation = LegacyQueueManager.Implementation.DISTRIBUTED_SNS;
}
else if ( "DISTRIBUTED".equals( queueManagerType ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java
index e220650..b2cebaa 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java
@@ -34,7 +34,7 @@ import java.util.Properties;
*/
public class KeyspaceDropper {
- private static final Logger logger = LoggerFactory.getLogger( AbstractTest.class );
+ private static final Logger logger = LoggerFactory.getLogger( KeyspaceDropper.class );
static { dropTestKeyspaces(); }
@@ -57,6 +57,10 @@ public class KeyspaceDropper {
dropTestKeyspace( keyspaceApp, hosts, port );
dropTestKeyspace( keyspaceQueue, hosts, port );
+
+ // drop local test keyspaces
+ dropTestKeyspace(keyspaceApp + "_", hosts, port);
+ dropTestKeyspace(keyspaceQueue + "_", hosts, port);
}
public static void dropTestKeyspace( String keyspace, String[] hosts, int port ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java
index d4ed7ef..8da2180 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java
@@ -340,7 +340,7 @@ public class QueueMessageManagerTest extends AbstractAkkaTest {
if (qmm.getQueueDepth( queueName, available ) == numMessages) {
break;
}
- Thread.sleep( 500 );
+ Thread.sleep( 1000 );
}
Assert.assertEquals( numMessages, qmm.getQueueDepth( queueName, available ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index cbec81f..4b76ca4 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -18,7 +18,7 @@
cassandra.startup=external
cassandra.connections=30
cassandra.timeout.pool=20000
-cassandra.timeout=20000
+cassandra.timeout=25000
hystrix.threadpool.graph_user.coreSize=1200
[15/15] usergrid git commit: Merge branch 'collectionClearJob'
Posted by md...@apache.org.
Merge branch 'collectionClearJob'
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8e9eff8d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8e9eff8d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8e9eff8d
Branch: refs/heads/master
Commit: 8e9eff8d9340fc85b7d4b544fa542e388974525a
Parents: 36e28a8 11823f2
Author: Mike Dunker <md...@google.com>
Authored: Mon Aug 28 16:23:01 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Aug 28 16:23:01 2017 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +-
.../usergrid/corepersistence/CoreModule.java | 1 +
.../corepersistence/CpEntityManager.java | 3 +-
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../corepersistence/EntityManagerFig.java | 2 +-
.../asyncevents/AsyncEventQueueType.java | 35 ++
.../asyncevents/AsyncEventService.java | 7 +-
.../asyncevents/AsyncEventServiceImpl.java | 545 +++++++++----------
.../asyncevents/EventBuilder.java | 26 +-
.../asyncevents/EventBuilderImpl.java | 205 +++++--
.../asyncevents/model/EntityDeleteEvent.java | 27 +-
.../index/CollectionDeleteAction.java | 43 ++
.../index/CollectionDeleteRequestBuilder.java | 92 ++++
.../CollectionDeleteRequestBuilderImpl.java | 146 +++++
.../index/CollectionDeleteService.java | 108 ++++
.../index/CollectionDeleteServiceImpl.java | 299 ++++++++++
.../index/CollectionSettingsImpl.java | 1 -
.../index/IndexProcessorFig.java | 27 +
.../corepersistence/index/IndexService.java | 16 +-
.../corepersistence/index/IndexServiceImpl.java | 56 +-
.../corepersistence/index/ReIndexAction.java | 5 +-
.../index/ReIndexServiceImpl.java | 5 +-
.../read/traverse/AbstractReadGraphFilter.java | 42 +-
.../AbstractReadReverseGraphFilter.java | 41 +-
.../usergrid/persistence/EntityManager.java | 2 +-
.../org/apache/usergrid/persistence/Query.java | 17 +
.../persistence/entities/Application.java | 26 +
.../rx/EdgesFromSourceObservableIT.java | 2 +-
.../rx/EdgesToTargetObservableIT.java | 4 +-
.../persistence/CollectionDeleteTest.java | 266 +++++++++
.../usergrid/persistence/CoreSchemaManager.java | 16 +-
.../usergrid/persistence/RebuildIndexTest.java | 36 +-
.../core/src/test/resources/project.properties | 2 +-
.../resources/usergrid-custom-test.properties | 3 +-
.../actorsystem/ActorSystemManagerImpl.java | 3 +-
.../src/test/resources/dynamic-test.properties | 1 +
.../src/test/resources/usergrid-CHOP.properties | 1 +
.../src/test/resources/usergrid-UNIT.properties | 1 +
.../src/test/resources/usergrid.properties | 1 +
.../persistence/graph/GraphManager.java | 2 +-
.../graph/impl/GraphManagerImpl.java | 21 +-
.../graph/impl/SimpleSearchByEdgeType.java | 9 +
.../graph/impl/stage/NodeDeleteListener.java | 6 +-
.../impl/stage/NodeDeleteListenerImpl.java | 64 +--
.../graph/serialization/EdgesObservable.java | 6 +-
.../impl/EdgeMetadataSerializationV2Impl.java | 1 -
.../serialization/impl/EdgesObservableImpl.java | 22 +-
.../impl/TargetIdObservableImpl.java | 4 +-
.../impl/migration/EdgeDataMigrationImpl.java | 2 +-
.../graph/impl/NodeDeleteListenerTest.java | 8 +-
.../src/test/resources/usergrid-AWS.properties | 1 +
.../src/test/resources/usergrid-CHOP.properties | 1 +
.../src/test/resources/usergrid-UNIT.properties | 1 +
.../src/test/resources/usergrid.properties | 1 +
.../usergrid/persistence/index/EntityIndex.java | 12 +-
.../index/impl/EsEntityIndexImpl.java | 65 +--
.../persistence/index/query/Identifier.java | 3 +-
.../src/test/resources/dynamic-test.properties | 1 +
.../src/test/resources/usergrid-CHOP.properties | 1 +
.../src/test/resources/usergrid-UNIT.properties | 1 +
stack/corepersistence/queue/pom.xml | 18 +-
.../queues/impl/QueueSerializationImpl.java | 2 +-
.../persistence/queue/guice/QueueModule.java | 2 +-
.../persistence/qakka/KeyspaceDropper.java | 6 +-
.../qakka/core/QueueMessageManagerTest.java | 2 +-
.../queue/src/test/resources/qakka.properties | 1 +
stack/pom.xml | 8 +-
.../resources/usergrid-custom-test.properties | 2 +
stack/rest/pom.xml | 4 +-
.../rest/applications/CollectionResource.java | 130 ++++-
.../rest/applications/users/UserResource.java | 9 +
.../rest/management/users/UserResource.java | 16 +
.../apache/usergrid/rest/TestResource/error.jsp | 4 +-
.../apache/usergrid/rest/TestResource/test.jsp | 5 +-
.../ApplicationResource/authorize_form.jsp | 15 +-
.../applications/ApplicationResource/error.jsp | 4 +-
.../users/UserResource/activate.jsp | 4 +-
.../applications/users/UserResource/confirm.jsp | 4 +-
.../applications/users/UserResource/error.jsp | 4 +-
.../users/UserResource/resetpw_email_form.jsp | 7 +-
.../UserResource/resetpw_email_success.jsp | 4 +-
.../users/UserResource/resetpw_set_form.jsp | 9 +-
.../users/UserResource/resetpw_set_success.jsp | 4 +-
.../applications/users/UsersResource/error.jsp | 4 +-
.../users/UsersResource/resetpw_email_form.jsp | 5 +-
.../UsersResource/resetpw_email_success.jsp | 4 +-
.../ManagementResource/authorize_form.jsp | 15 +-
.../management/ManagementResource/error.jsp | 4 +-
.../OrganizationResource/activate.jsp | 4 +-
.../OrganizationResource/confirm.jsp | 4 +-
.../OrganizationResource/error.jsp | 4 +-
.../management/users/UserResource/activate.jsp | 4 +-
.../management/users/UserResource/confirm.jsp | 4 +-
.../management/users/UserResource/error.jsp | 4 +-
.../users/UserResource/resetpw_email_form.jsp | 5 +-
.../UserResource/resetpw_email_success.jsp | 4 +-
.../users/UserResource/resetpw_set_form.jsp | 8 +-
.../users/UserResource/resetpw_set_success.jsp | 4 +-
.../management/users/UsersResource/error.jsp | 4 +-
.../users/UsersResource/resetpw_email_form.jsp | 4 +-
.../UsersResource/resetpw_email_success.jsp | 4 +-
.../rest/src/test/resources/project.properties | 2 +-
.../resources/usergrid-custom-test.properties | 4 +-
stack/services/pom.xml | 4 +-
.../usergrid/management/ManagementService.java | 10 +-
.../cassandra/ManagementServiceImpl.java | 10 +
.../usergrid/services/ActivitiesServiceIT.java | 9 +-
.../usergrid/services/ServiceInvocationIT.java | 3 +
.../AbstractServiceNotificationIT.java | 4 +-
.../gcm/NotificationsServiceIT.java | 2 +-
.../src/test/resources/project.properties | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
stack/test-utils/pom.xml | 4 +-
.../src/test/resources/project.properties | 2 +-
stack/tools/pom.xml | 2 +-
stack/websocket/pom.xml | 2 +-
116 files changed, 2013 insertions(+), 752 deletions(-)
----------------------------------------------------------------------
[05/15] usergrid git commit: more cassandra.timeout increases for
tests
Posted by md...@apache.org.
more cassandra.timeout increases for tests
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4b916c4d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4b916c4d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4b916c4d
Branch: refs/heads/master
Commit: 4b916c4d39fa0018fbe01f8e0489b0f5d083e41e
Parents: 70de6fd
Author: Mike Dunker <md...@google.com>
Authored: Thu Aug 17 11:57:53 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 11:57:53 2017 -0700
----------------------------------------------------------------------
stack/core/src/test/resources/usergrid-custom-test.properties | 1 +
.../collection/src/test/resources/dynamic-test.properties | 1 +
.../collection/src/test/resources/usergrid-CHOP.properties | 1 +
.../collection/src/test/resources/usergrid-UNIT.properties | 1 +
.../collection/src/test/resources/usergrid.properties | 1 +
.../common/src/test/resources/usergrid-UNIT.properties | 2 +-
.../graph/src/test/resources/usergrid-AWS.properties | 1 +
.../graph/src/test/resources/usergrid-CHOP.properties | 1 +
.../graph/src/test/resources/usergrid-UNIT.properties | 1 +
stack/corepersistence/graph/src/test/resources/usergrid.properties | 1 +
.../queryindex/src/test/resources/dynamic-test.properties | 1 +
.../queryindex/src/test/resources/usergrid-CHOP.properties | 1 +
.../queryindex/src/test/resources/usergrid-UNIT.properties | 1 +
stack/corepersistence/queue/src/test/resources/qakka.properties | 1 +
stack/rest/src/test/resources/usergrid-custom-test.properties | 1 +
15 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index 8f9058d..e32b146 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -16,6 +16,7 @@
# with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml)
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
#Not a good number for real systems. Write shards should be 2x cluster size from our tests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
index ca1a51c..59b8d31 100644
--- a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
+++ b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
@@ -2,6 +2,7 @@
# safe dynamic property defaults for our testing via IDE or Maven
cassandra.connections=30
cassandra.timeout.pool=30000
+cassandra.timeout=30000
cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
index c82bf83..9e95663 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
@@ -20,6 +20,7 @@
cassandra.connections=30
cassandra.timeout.pool=30000
+cassandra.timeout=30000
cassandra.port=9160
# a comma delimited private IP address list to your chop cassandra cluster
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
index 82bea14..0253e6e 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
@@ -21,6 +21,7 @@
# purposely setting connections lower to slow down the activity on the cassandra server locally
cassandra.connections=5
cassandra.timeout.pool=60000
+cassandra.timeout=60000
cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid.properties b/stack/corepersistence/collection/src/test/resources/usergrid.properties
index e85b1ac..b7a6c11 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid.properties
@@ -33,6 +33,7 @@ collection.uniquevalues.authoritative.region=us-east
cassandra.connections=15
cassandra.timeout.pool=240000
+cassandra.timeout=240000
queue.num.actors=5
queue.sender.num.actors=5
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
index 0a43209..13b2937 100644
--- a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
@@ -24,7 +24,7 @@ cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
collections.keyspace=Usergrid_Collections
-cassandra.timeout=10000
+cassandra.timeout=20000
cassandra.embedded=true
collections.keyspace.strategy.options=replication_factor:1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
index 4dd17e2..63da408 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
@@ -1,6 +1,7 @@
# Keep nothing but overriding test defaults in here
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
cassandra.hosts=
#cassandra.hosts=localhost
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties
index ea4c3c7..9680d88 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties
@@ -2,6 +2,7 @@
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
# a comma delimited private IP address list to your chop cassandra cluster
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
index 96834e6..3cb5608 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
@@ -1,6 +1,7 @@
# Keep nothing but overriding test defaults in here
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/graph/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid.properties b/stack/corepersistence/graph/src/test/resources/usergrid.properties
index d744efe..f8b7a8d 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid.properties
@@ -3,3 +3,4 @@ usergrid.cluster_name=usergrid
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
index 2589a7f..aaa4653 100644
--- a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
+++ b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
@@ -2,6 +2,7 @@
# safe dynamic property defaults for our testing via IDE or Maven
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties
index 677df96..855a8b5 100644
--- a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties
+++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties
@@ -1,6 +1,7 @@
# These are for CHOP environment settings
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
# a comma delimited private IP address list to your chop cassandra cluster
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
index 67a9ab1..c0166e7 100644
--- a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
@@ -6,6 +6,7 @@ cassandra.port=9160
cassandra.cluster_name=Usergrid
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
collections.keyspace=Usergrid_Collections
collections.keyspace.strategy.options=replication_factor:1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/queue/src/test/resources/qakka.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/resources/qakka.properties b/stack/corepersistence/queue/src/test/resources/qakka.properties
index d77e7e8..2a1a83e 100644
--- a/stack/corepersistence/queue/src/test/resources/qakka.properties
+++ b/stack/corepersistence/queue/src/test/resources/qakka.properties
@@ -67,3 +67,4 @@ cassandra.keyspace-drop-and-create=true
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 615bedd..cbec81f 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -18,6 +18,7 @@
cassandra.startup=external
cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
hystrix.threadpool.graph_user.coreSize=1200
[07/15] usergrid git commit: add a little more wait time
Posted by md...@apache.org.
add a little more wait time
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/25f1b0e5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/25f1b0e5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/25f1b0e5
Branch: refs/heads/master
Commit: 25f1b0e5a86f51d0da465d78cb0c5fbd1a006a7d
Parents: d06dbc9
Author: Mike Dunker <md...@google.com>
Authored: Wed Aug 16 22:30:34 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 11:59:53 2017 -0700
----------------------------------------------------------------------
.../apache/usergrid/rest/applications/queries/QueryTestBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25f1b0e5/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
index 0adafef..1eec9ae 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
@@ -65,7 +65,7 @@ public class QueryTestBase extends AbstractRestIT {
logger.info(entities[i].entrySet().toString());
}
//refresh the index so that they are immediately searchable
- this.waitForQueueDrainAndRefreshIndex();
+ this.waitForQueueDrainAndRefreshIndex(1000);
return entities;
}
[12/15] usergrid git commit: Converts de-indexing of edges and
entities to identify documents in the index to delete based on data in
Cassandra vs. resource intensive queries to the index. Fixes issue where
nodes were not actually getting deleted from gr
Posted by md...@apache.org.
Converts de-indexing of edges and entities to identify documents in the index to delete based on data in Cassandra vs. resource intensive queries to the index. Fixes issue where nodes were not actually getting deleted from graph as marked edges were being filtered out during the delete process itself. Update to a newer vs. of jamm (used for jvm memory management in the test framework).
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c201f1f6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c201f1f6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c201f1f6
Branch: refs/heads/master
Commit: c201f1f67aa0f373958c3886c893979215cba88f
Parents: 570e1ab
Author: Michael Russo <ru...@google.com>
Authored: Sat Jul 8 17:10:25 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Fri Aug 18 17:00:43 2017 -0700
----------------------------------------------------------------------
.../corepersistence/EntityManagerFig.java | 2 +-
.../asyncevents/AsyncEventServiceImpl.java | 23 +--
.../asyncevents/EventBuilder.java | 15 +-
.../asyncevents/EventBuilderImpl.java | 172 ++++++++++++++-----
.../corepersistence/index/IndexService.java | 16 +-
.../corepersistence/index/IndexServiceImpl.java | 56 ++----
.../read/traverse/AbstractReadGraphFilter.java | 37 +---
.../AbstractReadReverseGraphFilter.java | 36 +---
.../org/apache/usergrid/persistence/Query.java | 17 ++
.../rx/EdgesFromSourceObservableIT.java | 2 +-
.../rx/EdgesToTargetObservableIT.java | 4 +-
.../core/src/test/resources/project.properties | 2 +-
.../persistence/graph/GraphManager.java | 2 +-
.../graph/impl/GraphManagerImpl.java | 21 +--
.../graph/impl/SimpleSearchByEdgeType.java | 9 +
.../graph/impl/stage/NodeDeleteListener.java | 6 +-
.../impl/stage/NodeDeleteListenerImpl.java | 64 +++----
.../graph/serialization/EdgesObservable.java | 6 +-
.../impl/EdgeMetadataSerializationV2Impl.java | 1 -
.../serialization/impl/EdgesObservableImpl.java | 22 +--
.../impl/TargetIdObservableImpl.java | 4 +-
.../impl/migration/EdgeDataMigrationImpl.java | 2 +-
.../graph/impl/NodeDeleteListenerTest.java | 8 +-
.../usergrid/persistence/index/EntityIndex.java | 12 +-
.../index/impl/EsEntityIndexImpl.java | 63 -------
.../persistence/index/query/Identifier.java | 3 +-
stack/pom.xml | 8 +-
stack/rest/pom.xml | 4 +-
.../rest/src/test/resources/project.properties | 2 +-
stack/services/pom.xml | 4 +-
.../src/test/resources/project.properties | 2 +-
stack/test-utils/pom.xml | 4 +-
.../src/test/resources/project.properties | 2 +-
stack/tools/pom.xml | 2 +-
stack/websocket/pom.xml | 2 +-
35 files changed, 286 insertions(+), 349 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 872ffbb..46c7a1d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@ -39,7 +39,7 @@ public interface EntityManagerFig extends GuicyFig {
int sleep();
@Key( "usergrid.entityManager.enable_deindex_on_update" )
- @Default( "true" )
+ @Default( "false" )
boolean getDeindexOnUpdate();
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 0e55e9b..79a80c0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -640,8 +640,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
// default this observable's return to empty index operation message if nothing is emitted
- return eventBuilder.buildDeleteEdge(applicationScope, edge)
- .toBlocking().lastOrDefault(new IndexOperationMessage());
+ return eventBuilder.buildDeleteEdge(applicationScope, edge);
}
@@ -832,20 +831,11 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final ApplicationScope applicationScope = entityDeleteEvent.getEntityIdScope().getApplicationScope();
final Id entityId = entityDeleteEvent.getEntityIdScope().getId();
- if (logger.isDebugEnabled())
+ if (logger.isDebugEnabled()) {
logger.debug("Deleting entity id from index in app scope {} with entityId {}", applicationScope, entityId);
+ }
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete( applicationScope, entityId );
-
-
- // Delete the entities and remove from graph separately
- entityDeleteResults.getEntitiesDeleted().toBlocking().lastOrDefault(null);
-
- entityDeleteResults.getCompactedNode().toBlocking().lastOrDefault(null);
-
- // default this observable's return to empty index operation message if nothing is emitted
- return entityDeleteResults.getIndexObservable().toBlocking().lastOrDefault(new IndexOperationMessage());
+ return eventBuilder.buildEntityDelete( applicationScope, entityId );
}
@@ -885,9 +875,6 @@ public class AsyncEventServiceImpl implements AsyncEventService {
startWorker(AsyncEventQueueType.UTILITY);
}
- for (int i = 0; i < deleteCount; i++) {
- startWorker(AsyncEventQueueType.DELETE);
- }
if( indexQueue instanceof SNSQueueManagerImpl) {
logger.info("Queue manager implementation supports dead letters, start dead letter queue workers.");
for (int i = 0; i < indexDeadCount; i++) {
@@ -902,7 +889,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
startDeadQueueWorker(AsyncEventQueueType.DELETE);
}
}else{
- logger.info("Queue manager implementation does NOT support dead letters, NOT starting dead letter queue worker.");
+ logger.info("Queue manager implementation does NOT support dead letters, NOT starting dead letter queue workers.");
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
index 4db9f4b..ebb9190 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
@@ -27,6 +27,7 @@ import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -54,7 +55,7 @@ public interface EventBuilder {
* @param edge
* @return
*/
- Observable<IndexOperationMessage> buildDeleteEdge( ApplicationScope applicationScope, Edge edge );
+ IndexOperationMessage buildDeleteEdge( ApplicationScope applicationScope, Edge edge );
/**
* Return a bin with 2 observable streams for entity delete.
@@ -62,7 +63,7 @@ public interface EventBuilder {
* @param entityId
* @return
*/
- EntityDeleteResults buildEntityDelete(ApplicationScope applicationScope, Id entityId );
+ IndexOperationMessage buildEntityDelete(ApplicationScope applicationScope, Id entityId );
@@ -94,17 +95,17 @@ public interface EventBuilder {
- private final Observable<Id> compactedNode;
+ private final Observable<MarkedEdge> deletedEdges;
public EntityDeleteResults( final Observable<IndexOperationMessage> indexOperationMessageObservable,
final Observable<List<MvccLogEntry>> entitiesDeleted,
- final Observable<Id> compactedNode) {
+ final Observable<MarkedEdge> deletedEdges) {
this.indexOperationMessageObservable = indexOperationMessageObservable;
this.entitiesDeleted = entitiesDeleted;
- this.compactedNode = compactedNode;
+ this.deletedEdges = deletedEdges;
}
@@ -116,8 +117,8 @@ public interface EventBuilder {
return entitiesDeleted;
}
- public Observable<Id> getCompactedNode() {
- return compactedNode;
+ public Observable<MarkedEdge> getEdgesDeleted() {
+ return deletedEdges;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index bbdce5a..5051598 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -24,12 +24,11 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import org.apache.usergrid.corepersistence.index.*;
import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
-import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
@@ -82,7 +81,7 @@ public class EventBuilderImpl implements EventBuilder {
if (logger.isDebugEnabled()) {
logger.debug("Indexing in app scope {} with entity {} and new edge {}",
- applicationScope, entity, newEdge);
+ applicationScope, entity, newEdge);
}
return indexService.indexEdge( applicationScope, entity, newEdge );
@@ -90,15 +89,38 @@ public class EventBuilderImpl implements EventBuilder {
@Override
- public Observable<IndexOperationMessage> buildDeleteEdge( final ApplicationScope applicationScope, final Edge
+ public IndexOperationMessage buildDeleteEdge( final ApplicationScope applicationScope, final Edge
edge ) {
if (logger.isDebugEnabled()) {
logger.debug("Deleting in app scope {} with edge {}", applicationScope, edge);
}
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- return gm.deleteEdge( edge )
- .flatMap( deletedEdge -> indexService.deleteIndexEdge( applicationScope, deletedEdge ));
+ final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+ final IndexOperationMessage combined = new IndexOperationMessage();
+
+ gm.deleteEdge( edge )
+ .doOnNext( deletedEdge -> {
+
+ logger.debug("Processing deleted edge for de-indexing {}", deletedEdge);
+
+ // get ALL versions of the target node as any connection from this source node needs to be removed
+ ecm.getVersionsFromMaxToMin(deletedEdge.getTargetNode(), UUIDUtils.newTimeUUID())
+ .doOnNext(mvccLogEntry -> {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Adding edge {} mvccLogEntry {} to de-index batch", deletedEdge.getTargetNode(), mvccLogEntry);
+ }
+ combined.ingest(
+ indexService
+ .deIndexEdge(applicationScope, deletedEdge, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion())
+ .toBlocking().lastOrDefault(new IndexOperationMessage()));
+
+ }).toBlocking().lastOrDefault(null);
+
+ }).toBlocking().lastOrDefault(null);
+
+ return combined;
}
@@ -106,43 +128,102 @@ public class EventBuilderImpl implements EventBuilder {
//it'll need to be pushed up higher so we can do the marking that isn't async or does it not matter?
@Override
- public EntityDeleteResults buildEntityDelete(final ApplicationScope applicationScope, final Id entityId ) {
+ public IndexOperationMessage buildEntityDelete(final ApplicationScope applicationScope, final Id entityId ) {
+
if (logger.isDebugEnabled()) {
- logger.debug("Deleting entity id from index in app scope {} with entityId {}", applicationScope, entityId);
+ logger.debug("Deleting entity id (marked versions) from index in app scope {} with entityId {}",
+ applicationScope, entityId);
}
final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- //TODO USERGRID-1123: Implement so we don't iterate logs twice (latest DELETED version, then to get all DELETED)
-
- MvccLogEntry mostRecentlyMarked = ecm.getVersionsFromMaxToMin( entityId, UUIDUtils.newTimeUUID() ).toBlocking()
- .firstOrDefault( null, mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
-
- // De-indexing and entity deletes don't check log entries. We must do that first. If no DELETED logs, then
- // return an empty observable as our no-op.
- Observable<IndexOperationMessage> deIndexObservable = Observable.empty();
- Observable<List<MvccLogEntry>> ecmDeleteObservable = Observable.empty();
-
- if(mostRecentlyMarked != null){
+ MvccLogEntry mostRecentToDelete =
+ ecm.getVersionsFromMaxToMin( entityId, UUIDUtils.newTimeUUID() )
+ .toBlocking()
+ .firstOrDefault( null, mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
- // fetch entity versions to be de-index by looking in cassandra
- deIndexObservable =
- indexService.deIndexEntity(applicationScope, entityId, mostRecentlyMarked.getVersion(),
- getVersionsOlderThanMarked(ecm, entityId, mostRecentlyMarked.getVersion()));
+// logger.info("mostRecent stage={} entityId={} version={} state={}",
+// mostRecentToDelete.getStage().name(), mostRecentToDelete.getEntityId(),
+// mostRecentToDelete.getVersion().toString(), mostRecentToDelete.getState().name());
- ecmDeleteObservable =
- ecm.getVersionsFromMaxToMin( entityId, mostRecentlyMarked.getVersion() )
- .filter( mvccLogEntry->
- mvccLogEntry.getVersion().timestamp() <= mostRecentlyMarked.getVersion().timestamp() )
- .buffer( serializationFig.getBufferSize() )
- .doOnNext( buffer -> ecm.delete( buffer ) );
+ if (mostRecentToDelete == null) {
+ logger.info("No entity versions to delete for id {}", entityId.toString());
+ }
+ // if nothing is marked, then abort
+ if(mostRecentToDelete == null){
+ return new IndexOperationMessage();
}
- // Graph compaction checks the versions inside compactNode, just build this up for the caller to subscribe to
- final Observable<Id> graphCompactObservable = gm.compactNode(entityId);
-
- return new EntityDeleteResults( deIndexObservable, ecmDeleteObservable, graphCompactObservable );
+ final List<MvccLogEntry> logEntries = new ArrayList<>();
+ Observable<MvccLogEntry> mvccLogEntryListObservable =
+ ecm.getVersionsFromMaxToMin( entityId, UUIDUtils.newTimeUUID() );
+ mvccLogEntryListObservable
+ .filter( mvccLogEntry-> mvccLogEntry.getVersion().timestamp() <= mostRecentToDelete.getVersion().timestamp() )
+ .buffer( serializationFig.getBufferSize() )
+ .doOnNext( buffer -> ecm.delete( buffer ) )
+ .doOnNext(mvccLogEntries -> {
+ logEntries.addAll(mvccLogEntries);
+ }).toBlocking().lastOrDefault(null);
+
+ //logger.info("logEntries size={}", logEntries.size());
+
+ IndexOperationMessage combined = new IndexOperationMessage();
+
+ // do the edge deletes and build up de-index messages for each edge deleted
+ // assume we have "server1" and "region1" nodes in the graph with the following relationships (edges/connections):
+ //
+ // region1 -- zzzconnzzz|has --> server1
+ // server1 -- zzzconnzzz|in --> region1
+ //
+ // there will always be a relationship from the appId to each entity based on the entity type (collection):
+ //
+ // application -- zzzcollzzz|servers --> server1
+ // application -- zzzcollzzz|regions --> region1
+ //
+ // When deleting either "server1" or "region1" entity, the connections should get deleted and de-indexed along
+ // with the entry for the entity itself in the collection. The above example should have at minimum 3 things to
+ // be de-indexed. There may be more as either "server1" or "region1" could have multiple versions.
+ //
+ // Further comments using the example of deleting "server1" from the above example.
+ gm.compactNode(entityId).doOnNext(markedEdge -> {
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Processing deleted edge for de-indexing {}", markedEdge);
+ }
+
+ // if the edge was for a connection where the entity to be deleted is the source node, we need to load
+ // the target node's versions so that all versions of connections to that entity can be de-indexed
+ // server1 -- zzzconnzzz|in --> region1
+ if(!markedEdge.getTargetNode().getType().equals(entityId.getType())){
+
+ // get ALL versions of the target node as any connection from this source node needs to be removed
+ ecm.getVersionsFromMaxToMin( markedEdge.getTargetNode(), UUIDUtils.newTimeUUID() )
+ .doOnNext(mvccLogEntry -> {
+ logger.debug("Adding edge {} mvccLogEntry {} to de-index batch", markedEdge, mvccLogEntry);
+ combined.ingest(
+ indexService
+ .deIndexEdge(applicationScope, markedEdge, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion())
+ .toBlocking().lastOrDefault(new IndexOperationMessage()));
+
+ }).toBlocking().lastOrDefault(null);
+
+ }else {
+
+ // for each version of the entity being deleted, de-index the connections where the entity is the target
+ // node ( application -- zzzcollzzz|servers --> server1 ) or (region1 -- zzzconnzzz|has --> server1)
+ logEntries.forEach(logEntry -> {
+ logger.debug("Adding edge {} mvccLogEntry {} to de-index batch", markedEdge, logEntry);
+ combined.ingest(
+ indexService
+ .deIndexEdge(applicationScope, markedEdge, logEntry.getEntityId(), logEntry.getVersion())
+ .toBlocking().lastOrDefault(new IndexOperationMessage()));
+ });
+ }
+
+ }).toBlocking().lastOrDefault(null);
+
+ return combined;
}
@Override
@@ -185,22 +266,22 @@ public class EventBuilderImpl implements EventBuilder {
return indexService.deIndexOldVersions( applicationScope, entityId,
- getVersionsOlderThanMarked(ecm, entityId, markedVersion), markedVersion);
+ getVersionsOlderThanOrEqualToMarked(ecm, entityId, markedVersion));
}
- private List<UUID> getVersionsOlderThanMarked( final EntityCollectionManager ecm,
- final Id entityId, final UUID markedVersion ){
+ private List<UUID> getVersionsOlderThanOrEqualToMarked(final EntityCollectionManager ecm,
+ final Id entityId, final UUID markedVersion ){
final List<UUID> versions = new ArrayList<>();
- // only take last 5 versions to avoid eating memory. a tool can be built for massive cleanups for old usergrid
+ // only take last 100 versions to avoid eating memory. a tool can be built for massive cleanups for old usergrid
// clusters that do not have this in-line cleanup
ecm.getVersionsFromMaxToMin( entityId, markedVersion)
- .take(5)
+ .take(100)
.forEach( mvccLogEntry -> {
- if ( mvccLogEntry.getVersion().timestamp() < markedVersion.timestamp() ) {
+ if ( mvccLogEntry.getVersion().timestamp() <= markedVersion.timestamp() ) {
versions.add(mvccLogEntry.getVersion());
}
@@ -210,4 +291,17 @@ public class EventBuilderImpl implements EventBuilder {
return versions;
}
+ private List<UUID> getAllVersions( final EntityCollectionManager ecm,
+ final Id entityId ) {
+
+ final List<UUID> versions = new ArrayList<>();
+
+ ecm.getVersionsFromMaxToMin(entityId, UUIDUtils.newTimeUUID())
+ .forEach( mvccLogEntry -> {
+ versions.add(mvccLogEntry.getVersion());
+ });
+
+ return versions;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
index b989a9c..58d470a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -69,18 +69,17 @@ public interface IndexService {
*/
Observable<IndexOperationMessage> deleteIndexEdge(final ApplicationScope applicationScope, final Edge edge);
-
/**
- * De-index all documents with the specified entityId and versions provided. This will also remove any documents
- * where the entity is a source/target node ( index docs where this entityId is a part of connections).
- *
+ * Delete an index edge from the specified scope for a specific entity version
* @param applicationScope
+ * @param edge
* @param entityId
- * @param markedVersion
+ * @param entityVersion
* @return
*/
- Observable<IndexOperationMessage> deIndexEntity(final ApplicationScope applicationScope, final Id entityId,
- final UUID markedVersion, final List<UUID> allVersionsBeforeMarked);
+ Observable<IndexOperationMessage> deIndexEdge(final ApplicationScope applicationScope, final Edge edge,
+ final Id entityId, final UUID entityVersion);
+
/**
@@ -88,10 +87,9 @@ public interface IndexService {
*
* @param applicationScope
* @param entityId
- * @param markedVersion
* @return
*/
Observable<IndexOperationMessage> deIndexOldVersions(final ApplicationScope applicationScope, final Id entityId,
- final List<UUID> versions, UUID markedVersion);
+ final List<UUID> versions);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 1b8614f..32470f6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -29,9 +29,7 @@ import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.*;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.index.*;
@@ -103,7 +101,7 @@ public class IndexServiceImpl implements IndexService {
//we always index in the target scope
- final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId );
+ final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId, true);
//we may have to index we're indexing from source->target here
final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeFromSource( edge ) );
@@ -216,10 +214,8 @@ public class IndexServiceImpl implements IndexService {
return Optional.of(defaultProperties);
}
- //Steps to delete an IndexEdge.
- //1.Take the search edge given and search for all the edges in elasticsearch matching that search edge
- //2. Batch Delete all of those edges returned in the previous search.
- //TODO: optimize loops further.
+ // DO NOT USE THIS AS THE QUERY TO ES CAN CAUSE EXTREME LOAD
+ // TODO REMOVE THIS AND UPDATE THE TESTS TO NOT USE THIS METHOD
@Override
public Observable<IndexOperationMessage> deleteIndexEdge( final ApplicationScope applicationScope,
final Edge edge ) {
@@ -256,49 +252,31 @@ public class IndexServiceImpl implements IndexService {
return ObservableTimer.time( batches, addTimer );
}
-
@Override
- public Observable<IndexOperationMessage> deIndexEntity( final ApplicationScope applicationScope, final Id entityId,
- final UUID markedVersion,
- final List<UUID> allVersionsBeforeMarked ) {
-
- final EntityIndex ei = entityIndexFactory.
- createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
-
- // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code foe de-indexing
- // previously .timstamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data)
- final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
- CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId,
- Long.MAX_VALUE ) );
-
-
+ public Observable<IndexOperationMessage> deIndexEdge(final ApplicationScope applicationScope, final Edge edge,
+ final Id entityId, final UUID entityVersion){
- final EntityIndexBatch batch = ei.createBatch();
-
- // de-index each version of the entity before the marked version
- allVersionsBeforeMarked.forEach(version -> batch.deindex(searchEdgeFromSource, entityId, version));
-
-
- // for now, query the index to remove docs where the entity is source/target node and older than markedVersion
- // TODO: investigate getting this information from graph
- CandidateResults candidateResults = ei.getNodeDocsOlderThanMarked(entityId, markedVersion );
- candidateResults.forEach(candidateResult -> batch.deindex(candidateResult));
-
- return Observable.just(batch.build());
+ if (logger.isTraceEnabled()) {
+ logger.trace("deIndexEdge edge={} entityId={} entityVersion={}", edge.toString(), entityId.toString(), entityVersion.toString());
+ }
+ final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope));
+ final EntityIndexBatch entityBatch = ei.createBatch();
+ entityBatch.deindex(generateScopeFromSource( edge ), entityId, entityVersion);
+ return Observable.just(entityBatch.build());
}
+
@Override
public Observable<IndexOperationMessage> deIndexOldVersions(final ApplicationScope applicationScope,
final Id entityId,
- final List<UUID> versions,
- UUID markedVersion) {
+ final List<UUID> versions) {
final EntityIndex ei = entityIndexFactory.
createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
- // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code foe de-indexing
- // previously .timstamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data)
+ // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code for de-indexing
+ // previously .timsetamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data)
final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId,
Long.MAX_VALUE ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 4886b08..b1b7f75 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -122,12 +122,8 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
if (isDeleted) {
logger.info("Edge {} is deleted when seeking, deleting the edge", markedEdge);
- final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
-
- indexMessageObservable
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
@@ -136,18 +132,8 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
final Id sourceNodeId = markedEdge.getSourceNode();
logger.info("Edge {} has a deleted source node, deleting the entity for id {}", markedEdge, sourceNodeId);
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
-
- entityDeleteResults.getIndexObservable()
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
-
- Observable.merge(entityDeleteResults.getEntitiesDeleted(),
- entityDeleteResults.getCompactedNode())
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
- subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
@@ -156,19 +142,8 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
final Id targetNodeId = markedEdge.getTargetNode();
logger.info("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId);
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
-
- entityDeleteResults.getIndexObservable()
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
-
- Observable.merge(entityDeleteResults.getEntitiesDeleted(),
- entityDeleteResults.getCompactedNode())
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
- subscribe();
-
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
index e8a20dd..c75545e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
@@ -122,12 +122,8 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
if (isDeleted) {
logger.info("Edge {} is deleted when seeking, deleting the edge", markedEdge);
- final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
-
- indexMessageObservable
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
@@ -136,18 +132,8 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
final Id sourceNodeId = markedEdge.getSourceNode();
logger.info("Edge {} has a deleted source node, deleting the entity for id {}", markedEdge, sourceNodeId);
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
-
- entityDeleteResults.getIndexObservable()
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
-
- Observable.merge(entityDeleteResults.getEntitiesDeleted(),
- entityDeleteResults.getCompactedNode())
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
- subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
@@ -156,18 +142,8 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
final Id targetNodeId = markedEdge.getTargetNode();
logger.info("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId);
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
-
- entityDeleteResults.getIndexObservable()
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
-
- Observable.merge(entityDeleteResults.getEntitiesDeleted(),
- entityDeleteResults.getCompactedNode())
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
- subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 900bda5..4c9d16c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -31,6 +31,8 @@ import org.apache.usergrid.persistence.index.query.tree.Operand;
import org.apache.usergrid.persistence.index.utils.ClassUtils;
import org.apache.usergrid.persistence.index.utils.ListUtils;
import org.apache.usergrid.persistence.index.utils.MapUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Serializable;
@@ -43,6 +45,7 @@ import java.util.Map.Entry;
public class Query {
+ private static final Logger logger = LoggerFactory.getLogger(Query.class);
public enum Level {
IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
@@ -319,6 +322,13 @@ public class Query {
public static Query fromIdentifier( Object id ) {
+ if (id == null) {
+ throw new IllegalArgumentException("null identifier passed in");
+ }
+ Identifier objectIdentifier = Identifier.from(id);
+ if (objectIdentifier == null) {
+ throw new IllegalArgumentException("Supplied id results in null Identifier");
+ }
Query q = new Query();
q.addIdentifier( Identifier.from(id) );
return q;
@@ -409,6 +419,10 @@ public class Query {
}
for ( Identifier identifier : identifiers ) {
+ if (identifier == null) {
+ logger.error("containsUuidIdentifiersOnly(): identifier in identifiers list is null");
+ return false;
+ }
if ( !identifier.isUUID() ) {
return false;
}
@@ -635,6 +649,9 @@ public class Query {
if ( identifiers == null ) {
identifiers = new ArrayList<Identifier>();
}
+ if (identifier == null) {
+ throw new IllegalArgumentException("adding null identifier is not allowed");
+ }
identifiers.add( identifier );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java
index 3bfe460..68834b3 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java
@@ -92,7 +92,7 @@ public class EdgesFromSourceObservableIT extends AbstractCoreIT {
final GraphManager gm = managerCache.getGraphManager( scope );
- edgesToTargetObservable.edgesToTarget( gm, target ).doOnNext( new Action1<Edge>() {
+ edgesToTargetObservable.edgesToTarget( gm, target, true).doOnNext(new Action1<Edge>() {
@Override
public void call( final Edge edge ) {
final String edgeType = edge.getType();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
index 9e84219..55b77c0 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
@@ -89,7 +89,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
final GraphManager gm = managerCache.getGraphManager( scope );
- edgesFromSourceObservable.edgesFromSourceDescending( gm, applicationId ).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceDescending( gm, applicationId, true).doOnNext(edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
@@ -118,7 +118,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
//test connections
- edgesFromSourceObservable.edgesFromSourceDescending( gm, source ).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceDescending( gm, source, true).doOnNext(edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/project.properties b/stack/core/src/test/resources/project.properties
index 1a848bc..77a785a 100644
--- a/stack/core/src/test/resources/project.properties
+++ b/stack/core/src/test/resources/project.properties
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
target.directory=${project.build.directory}
-jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+jamm.path=-javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
index 000c633..b746c61 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
@@ -93,7 +93,7 @@ public interface GraphManager extends CPManager {
* @param node The node to remove. This will apply a timestamp to apply the delete + compact operation. Any edges connected to this node with a timestamp
* <= the specified time on the mark will be removed from the graph
*/
- Observable<Id> compactNode( final Id node );
+ Observable<MarkedEdge> compactNode( final Id node );
/**
* Get all versions of this edge where versions <= max version
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 5fcdcb4..2fe40b1 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -262,24 +262,17 @@ public class GraphManagerImpl implements GraphManager {
@Override
- public Observable<Id> compactNode( final Id inputNode ) {
-
+ public Observable<MarkedEdge> compactNode( final Id inputNode ) {
final UUID startTime = UUIDGenerator.newTimeUUID();
-
- final Observable<Id> nodeObservable =
- Observable.just( inputNode ).map( node -> nodeSerialization.getMaxVersion( scope, node ) ).takeWhile(
- maxTimestamp -> maxTimestamp.isPresent() )
-
+ final Observable<MarkedEdge> nodeObservable =
+ Observable.just( inputNode )
+ .map( node -> nodeSerialization.getMaxVersion( scope, node ) )
+ //.doOnNext(maxTimestamp -> logger.info("compactNode maxTimestamp={}", maxTimestamp.toString()))
+ .takeWhile(maxTimestamp -> maxTimestamp.isPresent() )
//map our delete listener
- .flatMap( timestamp -> nodeDeleteListener.receive( scope, inputNode, startTime ) )
- //set to 0 if nothing is emitted
- .lastOrDefault( 0 )
- //log for posterity
- .doOnNext( count -> logger.trace( "Removed {} edges from node {}", count, inputNode ) )
- //return our id
- .map( count -> inputNode );
+ .flatMap( timestamp -> nodeDeleteListener.receive( scope, inputNode, startTime ) );
return ObservableTimer.time( nodeObservable, this.deleteNodeTimer );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
index 9392dbc..71d2f1d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
@@ -169,6 +169,15 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
return true;
}
+ @Override
+ public String toString(){
+ return "SimpleSearchByEdgeType{node="+node
+ +", type="+type
+ +", maxTimestamp="+maxTimestamp
+ +", order="+order
+ +", filterMarked="+filterMarked
+ +", last="+last+"}";
+ }
@Override
public int hashCode() {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
index 68569e5..3bcdc55 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
@@ -22,6 +22,8 @@ package org.apache.usergrid.persistence.graph.impl.stage;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
@@ -39,8 +41,8 @@ public interface NodeDeleteListener {
* @param node The node that was deleted
* @param timestamp The timestamp of the event
*
- * @return An observable that emits the total number of edges that have been removed with this node both as the
+ * @return An observable that emits the marked edges that have been removed with this node both as the
* target and source
*/
- Observable<Integer> receive( final ApplicationScope scope, final Id node, final UUID timestamp );
+ Observable<MarkedEdge> receive(final ApplicationScope scope, final Id node, final UUID timestamp );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index cd5b1a8..df4e5d5 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@ -95,53 +95,37 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
* @param node The node that was deleted
* @param timestamp The timestamp of the event
*
- * @return An observable that emits the total number of edges that have been removed with this node both as the
+ * @return An observable that emits the marked edges that have been removed with this node both as the
* target and source
*/
- public Observable<Integer> receive( final ApplicationScope scope, final Id node, final UUID timestamp ) {
+ public Observable<MarkedEdge> receive( final ApplicationScope scope, final Id node, final UUID timestamp ) {
return Observable.just( node )
//delete source and targets in parallel and merge them into a single observable
- .flatMap( new Func1<Id, Observable<Integer>>() {
- @Override
- public Observable<Integer> call( final Id node ) {
-
- final Optional<Long> maxVersion = nodeSerialization.getMaxVersion( scope, node );
-
- if (logger.isTraceEnabled()) {
- logger.trace("Node with id {} has max version of {}", node, maxVersion.orNull());
- }
-
-
- if ( !maxVersion.isPresent() ) {
- return Observable.empty();
- }
-
-
- //do all the delete, then when done, delete the node
- return doDeletes( node, scope, maxVersion.get(), timestamp ).count()
- //if nothing is ever emitted, emit 0 so that we know no operations took place.
- // Finally remove
- // the
- // target node in the mark
- .doOnCompleted( new Action0() {
- @Override
- public void call() {
- try {
- nodeSerialization.delete( scope, node, maxVersion.get()).execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to connect to casandra", e );
- }
- }
- } );
- }
- } ).defaultIfEmpty( 0 );
- }
+ .flatMap( id -> {
+ final Optional<Long> maxVersion = nodeSerialization.getMaxVersion( scope, node );
+
+ if (logger.isTraceEnabled()) {
+ logger.trace("Node with id {} has max version of {}", node, maxVersion.orNull());
+ }
+ if ( !maxVersion.isPresent() ) {
+ return Observable.empty();
+ }
+ // do all the edge deletes and then remove the marked node, return all edges just deleted
+ return
+ doDeletes( node, scope, maxVersion.get(), timestamp ).doOnCompleted( () -> {
+ try {
+ nodeSerialization.delete( scope, node, maxVersion.get()).execute();
+ } catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to cassandra", e );
+ }
+ });
+ });
+ }
/**
* Do the deletes
*/
@@ -162,7 +146,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageSerialization.getEdgesToTarget(scope,
- new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent()));
+ new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent(), false));
}
}));
@@ -174,7 +158,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageSerialization.getEdgesFromSource(scope,
- new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent()));
+ new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent(), false));
}
}));
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index 78a1d4b..5577bd0 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -36,9 +36,10 @@ public interface EdgesObservable {
* Return an observable of all edges from a source
* @param gm
* @param sourceNode
+ * @param filterMarked
* @return
*/
- Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode );
+ Observable<Edge> edgesFromSourceDescending(final GraphManager gm, final Id sourceNode, boolean filterMarked);
/**
@@ -54,9 +55,10 @@ public interface EdgesObservable {
* Return an observable of all edges to a target
* @param gm
* @param targetNode
+ * @param filterMarked
* @return
*/
- Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode);
+ Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode, boolean filterMarked);
/**
* Return an observable of all edges from a source node. Ordered ascending, from the startTimestamp if specified
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
index 1f81864..e9e2b28 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
@@ -400,7 +400,6 @@ public class EdgeMetadataSerializationV2Impl implements EdgeMetadataSerializatio
ValidationUtils.validateApplicationScope( scope );
GraphValidation.validateSearchEdgeType( search );
-
final Id applicationId = scope.getApplication();
final Id searchNode = search.getNode();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index 20efe42..9e0998d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -35,7 +35,6 @@ import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
import rx.Observable;
-import rx.functions.Func1;
/**
@@ -55,7 +54,7 @@ public class EdgesObservableImpl implements EdgesObservable {
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode ) {
+ public Observable<Edge> edgesFromSourceDescending(final GraphManager gm, final Id sourceNode, boolean filterMarked) {
final Observable<String> edgeTypes =
gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
@@ -67,7 +66,7 @@ public class EdgesObservableImpl implements EdgesObservable {
return gm.loadEdgesFromSource(
new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- Optional.<Edge>absent() ) );
+ Optional.<Edge>absent(), filterMarked ) );
} );
}
@@ -119,19 +118,16 @@ public class EdgesObservableImpl implements EdgesObservable {
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesToTarget( final GraphManager gm, final Id targetNode ) {
- final Observable<String> edgeTypes =
- gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( targetNode, null, null ) );
-
- return edgeTypes.flatMap( edgeType -> {
-
- if (logger.isTraceEnabled()) {
- logger.trace("Loading edges of edgeType {} to {}", edgeType, targetNode);
- }
+ public Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode, boolean filterMarked) {
+ return gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( targetNode, null, null ) )
+ .flatMap( edgeType -> {
+ if (logger.isTraceEnabled()) {
+ logger.trace("Loading edges of edgeType {} to {}", edgeType, targetNode);
+ }
return gm.loadEdgesToTarget(
new SimpleSearchByEdgeType( targetNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- Optional.<Edge>absent() ) );
+ Optional.<Edge>absent(), filterMarked ) );
} );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
index 6a08d46..69dd43b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.persistence.graph.serialization.impl;
import com.google.inject.Inject;
-import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
@@ -29,7 +28,6 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import rx.functions.Func1;
/**
* Emits the id of all nodes that are target nodes from the given source node
@@ -55,7 +53,7 @@ public class TargetIdObservableImpl implements TargetIdObservable {
public Observable<Id> getTargetNodes(final GraphManager gm, final Id sourceNode) {
//only search edge types that start with collections
- return edgesFromSourceObservable.edgesFromSourceDescending( gm, sourceNode ).map( edge -> {
+ return edgesFromSourceObservable.edgesFromSourceDescending( gm, sourceNode, true).map(edge -> {
final Id targetNode = edge.getTargetNode();
if (logger.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index 8eccdbd..1d4331c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -89,7 +89,7 @@ public class EdgeDataMigrationImpl implements DataMigration {
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
//get edges from the source
- return edgesFromSourceObservable.edgesFromSourceDescending( gm, graphNode.entryNode ).buffer( 1000 )
+ return edgesFromSourceObservable.edgesFromSourceDescending( gm, graphNode.entryNode, true).buffer( 1000 )
.doOnNext( edges -> {
final MutationBatch batch = keyspace.prepareMutationBatch();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
index 438a978..80198de 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
@@ -135,7 +135,7 @@ public class NodeDeleteListenerTest {
UUID eventTime = UUIDGenerator.newTimeUUID();
- int count = deleteListener.receive( scope, sourceNode, eventTime ).toBlocking().last();
+ int count = deleteListener.receive( scope, sourceNode, eventTime ).count().toBlocking().last();
assertEquals( "Mark was not set, no delete should be executed", 0, count );
@@ -171,7 +171,7 @@ public class NodeDeleteListenerTest {
nodeSerialization.mark( scope, sourceNode, timestamp ).execute();
- int count = deleteListener.receive( scope, sourceNode, deleteEventTimestamp ).toBlocking().last();
+ int count = deleteListener.receive( scope, sourceNode, deleteEventTimestamp ).count().toBlocking().last();
assertEquals( 1, count );
@@ -256,7 +256,7 @@ public class NodeDeleteListenerTest {
nodeSerialization.mark( scope, targetNode, deleteBefore ).execute();
- int count = deleteListener.receive( scope, targetNode, UUIDGenerator.newTimeUUID() ).toBlocking().last();
+ int count = deleteListener.receive( scope, targetNode, UUIDGenerator.newTimeUUID() ).count().toBlocking().last();
assertEquals( 1, count );
@@ -366,7 +366,7 @@ public class NodeDeleteListenerTest {
nodeSerialization.mark( scope, toDelete, deleteVersion ).execute();
- int count = deleteListener.receive( scope, toDelete, UUIDGenerator.newTimeUUID() ).toBlocking().last();
+ int count = deleteListener.receive( scope, toDelete, UUIDGenerator.newTimeUUID() ).count().toBlocking().last();
assertEquals( edgeCount, count );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 14020a9..b444199 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -20,14 +20,12 @@
package org.apache.usergrid.persistence.index;
-import com.google.common.base.Optional;
import org.apache.usergrid.persistence.core.CPManager;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
import java.util.Map;
-import java.util.UUID;
/**
@@ -36,7 +34,7 @@ import java.util.UUID;
public interface EntityIndex extends CPManager {
- public static final int MAX_LIMIT = 1000;
+ int MAX_LIMIT = 1000;
/**
* Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
@@ -134,14 +132,6 @@ public interface EntityIndex extends CPManager {
CandidateResults getAllEdgeDocuments(final IndexEdge edge, final Id entityId);
/**
- * Returns all entity docs that match the entityId being the nodeId ( aka connections where entityId = sourceNode)
- *
- * @param entityId The entityId to match when searching
- * @param markedVersion The version that has been marked for deletion. All version before this one must be deleted.
- * @return
- */
- CandidateResults getNodeDocsOlderThanMarked(final Id entityId, final UUID markedVersion);
- /**
* delete all application records
*
* @return
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index cb695d5..f4fae2b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -34,7 +34,6 @@ import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.StringUtils;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.ElasticSearchQueryBuilder.SearchRequestBuilderStrategyV2;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
@@ -584,68 +583,6 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
- @Override
- public CandidateResults getNodeDocsOlderThanMarked(final Id entityId, final UUID markedVersion ) {
-
- // TODO: investigate if functionality via iterator so a caller can page the deletion until all is gone
-
- Preconditions.checkNotNull( entityId, "entityId cannot be null" );
- Preconditions.checkNotNull(markedVersion, "markedVersion cannot be null");
- ValidationUtils.verifyVersion(markedVersion);
-
- SearchResponse searchResponse;
- List<CandidateResult> candidates = new ArrayList<>();
-
- final long markedTimestamp = markedVersion.timestamp();
-
- // never let this fetch more than 100 to save memory
- final int searchLimit = Math.min(100, indexFig.getVersionQueryLimit());
-
- // this query will find all the documents where this entity is a source/target node
- final QueryBuilder nodeQuery = QueryBuilders
- .termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(entityId));
-
- final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder()
- .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC);
-
- try {
-
- long queryTimestamp = 0L;
-
- QueryBuilder timestampQuery = QueryBuilders
- .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
- .gte(queryTimestamp)
- .lt(markedTimestamp);
-
- QueryBuilder finalQuery = QueryBuilders.constantScoreQuery(
- QueryBuilders
- .boolQuery()
- .must(timestampQuery)
- .must(nodeQuery)
- );
-
-
- searchResponse = srb
- .setQuery(finalQuery)
- .setSize(searchLimit)
- .execute()
- .actionGet();
-
-
- candidates = aggregateScrollResults(candidates, searchResponse, markedVersion);
-
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t.getMessage() );
- failureMonitor.fail( "Unable to execute batch", t );
- throw t;
- }
- failureMonitor.success();
-
- return new CandidateResults( candidates, Collections.EMPTY_SET);
- }
-
-
/**
* Completely delete an index.
*/
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
index 84a28f0..70d2284 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
@@ -34,6 +34,7 @@ public class Identifier implements Serializable {
public static final String UUID_REX =
"[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}";
public static final String EMAIL_REX = "[a-zA-Z0-9._%'+\\-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}";
+ public static final String NAME_REX = "[a-zA-Z0-9_\\-./'+ ]*";
public enum Type {
UUID, NAME, EMAIL
@@ -46,7 +47,7 @@ public class Identifier implements Serializable {
static Pattern emailRegEx = Pattern.compile( EMAIL_REX );
// "Pattern nameRegEx" below used to be [a-zA-Z0-9_\\-./], changed it to contain a 'space' to a
// ddress https://issues.apache.org/jira/browse/USERGRID-94
- static Pattern nameRegEx = Pattern.compile( "[a-zA-Z0-9_\\-./'+ ]*" );
+ static Pattern nameRegEx = Pattern.compile( NAME_REX );
private Identifier( Type type, Object value ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 513c894..c98c72d 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -661,9 +661,9 @@
</dependency>
<dependency>
- <groupId>com.github.stephenc</groupId>
+ <groupId>com.github.jbellis</groupId>
<artifactId>jamm</artifactId>
- <version>0.2.5</version>
+ <version>0.3.1</version>
</dependency>
<!-- Third Party Non-Commercial Dependencies -->
@@ -1311,7 +1311,7 @@
<useSystemClassLoader>false</useSystemClassLoader>
<testFailureIgnore>false</testFailureIgnore>
<argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin}
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
${ug.argline}
</argLine>
<systemPropertyVariables>
@@ -1548,7 +1548,7 @@
<version>${surefire.plugin.version}</version>
<configuration>
<argLine>
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
${ug.argline}
-javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec
</argLine>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/rest/pom.xml
----------------------------------------------------------------------
diff --git a/stack/rest/pom.xml b/stack/rest/pom.xml
index 9bb83a6..e7aa68a 100644
--- a/stack/rest/pom.xml
+++ b/stack/rest/pom.xml
@@ -93,7 +93,7 @@
<argLine>-Dwebapp.directory=${basedir}/src/main/webapp
-Dtest.barrier.timestamp=${maven.build.timestamp} -Dtest.clean.storage=true -Xmx${ug.heapmax}
-Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
-Djava.util.logging.config.file=${basedir}/src/test/resources/logging.properties ${ug.argline}
</argLine>
<includes>
@@ -460,7 +460,7 @@
-Dwebapp.directory=${basedir}/src/main/webapp
-Dtest.barrier.timestamp=${maven.build.timestamp} -Dtest.clean.storage=true
-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
-Djava.util.logging.config.file=${basedir}/src/test/resources/logging.properties
${ug.argline}
</argLine>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/rest/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/project.properties b/stack/rest/src/test/resources/project.properties
index 94ef3bd..20a38f6 100644
--- a/stack/rest/src/test/resources/project.properties
+++ b/stack/rest/src/test/resources/project.properties
@@ -15,4 +15,4 @@
# limitations under the License.
target.directory=${project.build.directory}
-jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+jamm.path=-javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index b1df1b4..29fa311 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -102,7 +102,7 @@
<useSystemClassLoader>false</useSystemClassLoader>
<argLine>-Dtest.barrier.timestamp=${maven.build.timestamp} -Dtest.clean.storage=true
-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
${ug.argline} -Dlog4j.configuration=file:${basedir}/src/test/resources/log4j.properties
</argLine>
<includes>
@@ -499,7 +499,7 @@
<configuration>
<argLine>-Dtest.barrier.timestamp=${maven.build.timestamp} -Dtest.clean.storage=true
-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
${ug.argline}
-javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec
</argLine>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/services/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/project.properties b/stack/services/src/test/resources/project.properties
index d38e878..03736c0 100644
--- a/stack/services/src/test/resources/project.properties
+++ b/stack/services/src/test/resources/project.properties
@@ -16,4 +16,4 @@
target.directory=${project.build.directory}
resources.dir=${project.build.directory}
-jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+jamm.path=-javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/stack/test-utils/pom.xml b/stack/test-utils/pom.xml
index bbcf1ff..f99d43a 100644
--- a/stack/test-utils/pom.xml
+++ b/stack/test-utils/pom.xml
@@ -59,7 +59,7 @@
<threadCount>${usergrid.it.threads}</threadCount>
<threadCountClasses></threadCountClasses>
<reuseForks>true</reuseForks>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.0/jamm-0.3.0.jar ${ug.argline}</argLine>
<includes>
<include>**/CassandraResourceITSuite.java</include>
</includes>
@@ -290,7 +290,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>${surefire.plugin.version}</version>
<configuration>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.0/jamm-0.3.0.jar ${ug.argline}</argLine>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/test-utils/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/test/resources/project.properties b/stack/test-utils/src/test/resources/project.properties
index cd5b819..0bc9bb7 100644
--- a/stack/test-utils/src/test/resources/project.properties
+++ b/stack/test-utils/src/test/resources/project.properties
@@ -14,4 +14,4 @@
# limitations under the License.
target.directory=${project.build.directory}
-jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+jamm.path=-javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/tools/pom.xml
----------------------------------------------------------------------
diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml
index cbd2c1e..b34b068 100644
--- a/stack/tools/pom.xml
+++ b/stack/tools/pom.xml
@@ -61,7 +61,7 @@
<storage-config>${basedir}/src/test/conf</storage-config>
</systemPropertyVariables>
<forkMode>always</forkMode>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.0/jamm-0.3.0.jar ${ug.argline}</argLine>
</configuration>
</plugin>
[14/15] usergrid git commit: Add ability to walk through a collection
and delete all the entities,
optionally up to a certain timestamp. Modeled after reindex services.
Posted by md...@apache.org.
Add ability to walk through a collection and delete all the entities, optionally up to a certain timestamp. Modeled after reindex services.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/11823f29
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/11823f29
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/11823f29
Branch: refs/heads/master
Commit: 11823f294dfae762754ef1c4da8a5ee573107968
Parents: 3f7afcd
Author: Mike Dunker <md...@google.com>
Authored: Mon Aug 28 14:46:17 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Aug 28 14:46:17 2017 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 1 +
.../asyncevents/AsyncEventService.java | 3 +-
.../asyncevents/AsyncEventServiceImpl.java | 67 +++--
.../asyncevents/EventBuilder.java | 13 +-
.../asyncevents/EventBuilderImpl.java | 45 ++-
.../asyncevents/model/EntityDeleteEvent.java | 27 +-
.../index/CollectionDeleteAction.java | 43 +++
.../index/CollectionDeleteRequestBuilder.java | 92 ++++++
.../CollectionDeleteRequestBuilderImpl.java | 146 +++++++++
.../index/CollectionDeleteService.java | 108 +++++++
.../index/CollectionDeleteServiceImpl.java | 299 +++++++++++++++++++
.../index/IndexProcessorFig.java | 9 +
.../index/ReIndexServiceImpl.java | 2 +-
.../persistence/CollectionDeleteTest.java | 266 +++++++++++++++++
.../resources/usergrid-custom-test.properties | 2 +
.../rest/applications/CollectionResource.java | 130 +++++++-
16 files changed, 1217 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index ec6b775..a0748e6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -140,6 +140,7 @@ public class CoreModule extends AbstractModule {
bind( ReIndexService.class ).to( ReIndexServiceImpl.class );
+ bind( CollectionDeleteService.class ).to( CollectionDeleteServiceImpl.class );
bind( ExportService.class ).to( ExportServiceImpl.class );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index 9e346cf..04eaf4c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.asyncevents;
+import org.apache.usergrid.corepersistence.index.CollectionDeleteAction;
import org.apache.usergrid.corepersistence.index.ReIndexAction;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
@@ -33,7 +34,7 @@ import java.util.UUID;
/**
* Low level queue service for events in the entity. These events are fire and forget, and will always be asynchronous
*/
-public interface AsyncEventService extends ReIndexAction {
+public interface AsyncEventService extends ReIndexAction, CollectionDeleteAction {
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 428772f..3d06cae 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -75,9 +75,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.apache.commons.lang.StringUtils.indexOf;
-import static org.apache.commons.lang.StringUtils.isNotEmpty;
-
/**
* TODO, this whole class is becoming a nightmare.
@@ -106,7 +103,6 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public static final String QUEUE_NAME = "index"; //keep this short as AWS limits queue name size to 80 chars
public static final String QUEUE_NAME_UTILITY = "utility"; //keep this short as AWS limits queue name size to 80 chars
public static final String QUEUE_NAME_DELETE = "delete";
- public static final String DEAD_LETTER_SUFFIX = "_dead";
private final LegacyQueueManager indexQueue;
@@ -522,8 +518,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
applicationScope);
- logger.trace("Offering InitializeApplicationIndexEvent for {}:{}",
- applicationScope.getApplication().getUuid(), applicationScope.getApplication().getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering InitializeApplicationIndexEvent for {}:{}",
+ applicationScope.getApplication().getUuid(), applicationScope.getApplication().getType());
+ }
offerTopic( new InitializeApplicationIndexEvent( queueFig.getPrimaryRegion(),
new ReplicatedIndexLocationStrategy( indexLocationStrategy ) ), AsyncEventQueueType.REGULAR);
@@ -535,8 +533,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final Entity entity, long updatedAfter) {
- logger.trace("Offering EntityIndexEvent for {}:{}",
- entity.getId().getUuid(), entity.getId().getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering EntityIndexEvent for {}:{}",
+ entity.getId().getUuid(), entity.getId().getType());
+ }
offer(new EntityIndexEvent(queueFig.getPrimaryRegion(),
new EntityIdScope(applicationScope, entity.getId()), updatedAfter));
@@ -577,8 +577,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final Entity entity,
final Edge newEdge) {
- logger.trace("Offering EdgeIndexEvent for edge type {} entity {}:{}",
- newEdge.getType(), entity.getId().getUuid(), entity.getId().getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering EdgeIndexEvent for edge type {} entity {}:{}",
+ newEdge.getType(), entity.getId().getUuid(), entity.getId().getType());
+ }
offer( new EdgeIndexEvent( queueFig.getPrimaryRegion(), applicationScope, entity.getId(), newEdge ));
@@ -612,8 +614,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public void queueDeleteEdge(final ApplicationScope applicationScope,
final Edge edge) {
- logger.trace("Offering EdgeDeleteEvent for type {} to target {}:{}",
- edge.getType(), edge.getTargetNode().getUuid(), edge.getTargetNode().getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering EdgeDeleteEvent for type {} to target {}:{}",
+ edge.getType(), edge.getTargetNode().getUuid(), edge.getTargetNode().getType());
+ }
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ), AsyncEventQueueType.DELETE );
@@ -675,7 +679,9 @@ public class AsyncEventServiceImpl implements AsyncEventService {
//send to the topic so all regions index the batch
- logger.trace("Offering ElasticsearchIndexEvent for message {}", newMessageId );
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering ElasticsearchIndexEvent for message {}", newMessageId);
+ }
offerTopic( elasticsearchIndexEvent, queueType );
}
@@ -749,8 +755,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// queue the de-index of old versions to the topic so cleanup happens in all regions
- logger.trace("Offering DeIndexOldVersionsEvent for app {} {}:{}",
- applicationScope.getApplication().getUuid(), entityId.getUuid(), entityId.getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering DeIndexOldVersionsEvent for app {} {}:{}",
+ applicationScope.getApplication().getUuid(), entityId.getUuid(), entityId.getType());
+ }
offerTopic( new DeIndexOldVersionsEvent( queueFig.getPrimaryRegion(),
new EntityIdScope( applicationScope, entityId), markedVersion), AsyncEventQueueType.DELETE );
@@ -810,7 +818,9 @@ public class AsyncEventServiceImpl implements AsyncEventService {
@Override
public void queueEntityDelete(final ApplicationScope applicationScope, final Id entityId) {
- logger.trace("Offering EntityDeleteEvent for {}:{}", entityId.getUuid(), entityId.getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering EntityDeleteEvent for {}:{}", entityId.getUuid(), entityId.getType());
+ }
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
offer( new EntityDeleteEvent(queueFig.getPrimaryRegion(), new EntityIdScope( applicationScope, entityId ) ),
@@ -830,12 +840,15 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final EntityDeleteEvent entityDeleteEvent = ( EntityDeleteEvent ) event;
final ApplicationScope applicationScope = entityDeleteEvent.getEntityIdScope().getApplicationScope();
final Id entityId = entityDeleteEvent.getEntityIdScope().getId();
+ final boolean isCollectionDelete = entityDeleteEvent.isCollectionDelete();
+ final long updatedBefore = entityDeleteEvent.getUpdatedBefore();
if (logger.isDebugEnabled()) {
- logger.debug("Deleting entity id from index in app scope {} with entityId {}", applicationScope, entityId);
+ logger.debug("Deleting entity id from index in app scope {} with entityId {}, isCollectionDelete {}, updatedBefore {}",
+ applicationScope, entityId, isCollectionDelete, updatedBefore);
}
- return eventBuilder.buildEntityDelete( applicationScope, entityId );
+ return eventBuilder.buildEntityDelete( applicationScope, entityId, isCollectionDelete, updatedBefore );
}
@@ -1192,11 +1205,27 @@ public class AsyncEventServiceImpl implements AsyncEventService {
});
- logger.trace("Offering batch of EntityIndexEvent of size {}", batch.size());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering batch of EntityIndexEvent of size {}", batch.size());
+ }
offerBatch( batch, queueType );
}
+ public void deleteBatch(final List<EdgeScope> edges, final long updatedBefore, AsyncEventQueueType queueType) {
+
+ final List<EntityDeleteEvent> batch = new ArrayList<>();
+ edges.forEach(e -> {
+
+ //change to id scope to avoid serialization issues
+ batch.add(new EntityDeleteEvent(queueFig.getPrimaryRegion(),
+ new EntityIdScope(e.getApplicationScope(), e.getEdge().getTargetNode()), true, updatedBefore));
+
+ });
+
+ offerBatch(batch, queueType);
+ }
+
public class IndexEventResult{
private final Optional<IndexOperationMessage> indexOperationMessage;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
index ebb9190..4bb6312 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
@@ -63,7 +63,18 @@ public interface EventBuilder {
* @param entityId
* @return
*/
- IndexOperationMessage buildEntityDelete(ApplicationScope applicationScope, Id entityId );
+ IndexOperationMessage buildEntityDelete(ApplicationScope applicationScope, Id entityId);
+
+ /**
+ * Return a bin with 2 observable streams for entity delete.
+ * @param applicationScope
+ * @param entityId
+ * @param isCollectionDelete
+ * @param updatedBefore
+ * @return
+ */
+ IndexOperationMessage buildEntityDelete(ApplicationScope applicationScope, Id entityId,
+ boolean isCollectionDelete, long updatedBefore);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 5051598..7c72b72 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -24,7 +24,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import org.antlr.misc.Graph;
import org.apache.usergrid.corepersistence.index.*;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -128,15 +130,48 @@ public class EventBuilderImpl implements EventBuilder {
//it'll need to be pushed up higher so we can do the marking that isn't async or does it not matter?
@Override
- public IndexOperationMessage buildEntityDelete(final ApplicationScope applicationScope, final Id entityId ) {
+ public IndexOperationMessage buildEntityDelete(final ApplicationScope applicationScope, final Id entityId) {
+ return buildEntityDelete(applicationScope, entityId, false, Long.MAX_VALUE);
+ }
+
+ @Override
+ public IndexOperationMessage buildEntityDelete(final ApplicationScope applicationScope, final Id entityId,
+ final boolean isCollectionDelete, final long updatedBefore) {
if (logger.isDebugEnabled()) {
- logger.debug("Deleting entity id (marked versions) from index in app scope {} with entityId {}",
- applicationScope, entityId);
+ logger.debug("Deleting entity id (marked versions) from index in app scope {} with entityId {}, isCollectionDelete {}, updatedBefore={}",
+ applicationScope, entityId, isCollectionDelete, updatedBefore);
+ }
+
+ final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager(applicationScope);
+ final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
+
+ boolean deleteEntity = ecm.load(entityId).
+ map(entity -> {
+ final Field<Long> modified = entity.getField( Schema.PROPERTY_MODIFIED );
+
+ boolean willDelete = false;
+ if ( modified == null ) {
+ // We don't have a modified field, so we can't check, so delete it
+ willDelete = true;
+ } else if (modified.getValue() <= updatedBefore) {
+ willDelete = true;
+ }
+
+ if (isCollectionDelete && willDelete) {
+ // need to mark for deletion
+ ecm.mark(entityId, null)
+ .mergeWith(gm.markNode(entityId, CpNamingUtils.createGraphOperationTimestamp()))
+ .toBlocking().last();
+ }
+
+ return willDelete;
+ }).toBlocking().firstOrDefault(true);
+
+ if (!deleteEntity) {
+ return new IndexOperationMessage();
}
- final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
- final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
MvccLogEntry mostRecentToDelete =
ecm.getVersionsFromMaxToMin( entityId, UUIDUtils.newTimeUUID() )
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java
index 01d2ba8..1589632 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java
@@ -24,6 +24,7 @@ import org.apache.usergrid.persistence.collection.serialization.impl.migration.E
/**
* Event that will signal to finish the actual delete (post-mark delete) for an Entity
+ * It will mark if this is for a collection delete
*/
public final class EntityDeleteEvent extends AsyncEvent {
@@ -31,17 +32,41 @@ public final class EntityDeleteEvent extends AsyncEvent {
@JsonProperty
protected EntityIdScope entityIdScope;
+ @JsonProperty
+ private long updatedBefore;
+
+ @JsonProperty
+ private boolean isCollectionDelete;
+
public EntityDeleteEvent() {
super();
}
public EntityDeleteEvent(String sourceRegion, EntityIdScope entityIdScope) {
super(sourceRegion);
- this.entityIdScope = entityIdScope;
+ this.entityIdScope = entityIdScope;
+ this.updatedBefore = Long.MAX_VALUE;
+ this.isCollectionDelete = false;
+ }
+
+ public EntityDeleteEvent(String sourceRegion, EntityIdScope entityIdScope,
+ boolean isCollectionDelete, long updatedBefore) {
+ super(sourceRegion);
+ this.entityIdScope = entityIdScope;
+ this.updatedBefore = updatedBefore;
+ this.isCollectionDelete = isCollectionDelete;
}
public EntityIdScope getEntityIdScope() {
return entityIdScope;
}
+
+ public long getUpdatedBefore() {
+ return updatedBefore;
+ }
+
+ public boolean isCollectionDelete() {
+ return isCollectionDelete;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteAction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteAction.java
new file mode 100644
index 0000000..7bad06b
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteAction.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import java.util.List;
+
+
+/**
+ * Callback to perform a collection delete operation based on an scope during bulk collection delete operations
+ */
+public interface CollectionDeleteAction {
+
+ /**
+ * Delete a batch list of entities.
+ * @param edges
+ * @param updatedBefore
+ * @param queueType
+ */
+ void deleteBatch(final List<EdgeScope> edges, final long updatedBefore, AsyncEventQueueType queueType);
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilder.java
new file mode 100644
index 0000000..4abdfea
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilder.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import com.google.common.base.Optional;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+
+/**
+ * A builder interface to build our collection delete request
+ */
+public interface CollectionDeleteRequestBuilder {
+
+ /**
+ * Set the application id
+ */
+ CollectionDeleteRequestBuilder withApplicationId(final UUID applicationId);
+
+ /**
+ * Set the collection name.
+ * @param collectionName
+ * @return
+ */
+ CollectionDeleteRequestBuilder withCollection(final String collectionName);
+
+ /**
+ * Set our cursor to resume processing
+ * @param cursor
+ * @return
+ */
+ CollectionDeleteRequestBuilder withCursor(final String cursor);
+
+
+ CollectionDeleteRequestBuilder withDelay(int delayTimer, TimeUnit timeUnit);
+
+ /**
+ * Set the timestamp to delete entities updated <= this timestamp
+ * @param timestamp
+ * @return
+ */
+ CollectionDeleteRequestBuilder withEndTimestamp(final Long timestamp);
+
+
+ Optional<Integer> getDelayTimer();
+
+ Optional<TimeUnit> getTimeUnitOptional();
+
+ /**
+ * Get the application scope
+ * @return
+ */
+ Optional<ApplicationScope> getApplicationScope();
+
+ /**
+ * Get the collection name
+ * @return
+ */
+ Optional<String> getCollectionName();
+
+ /**
+ * Get the cursor
+ * @return
+ */
+ Optional<String> getCursor();
+
+ /**
+ * Get the latest timestamp to delete
+ * @return
+ */
+ Optional<Long> getEndTimestamp();
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilderImpl.java
new file mode 100644
index 0000000..890b770
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilderImpl.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import com.google.common.base.Optional;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+
+/**
+ * collection delete service request builder
+ */
+public class CollectionDeleteRequestBuilderImpl implements CollectionDeleteRequestBuilder {
+
+ private Optional<UUID> withApplicationId = Optional.absent();
+ private Optional<String> withCollectionName = Optional.absent();
+ private Optional<String> cursor = Optional.absent();
+ private Optional<Long> endTimestamp = Optional.absent();
+ private Optional<Integer> delayTimer = Optional.absent();
+ private Optional<TimeUnit> timeUnitOptional = Optional.absent();
+
+
+ /***
+ *
+ * @param applicationId The application id
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withApplicationId( final UUID applicationId ) {
+ this.withApplicationId = Optional.fromNullable( applicationId );
+ return this;
+ }
+
+
+ /**
+ * the collection name
+ * @param collectionName
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withCollection( final String collectionName ) {
+ this.withCollectionName = Optional.fromNullable( CpNamingUtils.getEdgeTypeFromCollectionName( collectionName.toLowerCase() ) );
+ return this;
+ }
+
+
+ /**
+ * The cursor
+ * @param cursor
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withCursor( final String cursor ) {
+ this.cursor = Optional.fromNullable( cursor );
+ return this;
+ }
+
+
+ /**
+ * Determines whether we should tack on a delay for collection delete and for how long if we do. Also
+ * allowed to specify how throttled back it should be.
+ * @param delayTimer
+ * @param timeUnit
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withDelay( final int delayTimer, final TimeUnit timeUnit ){
+ this.delayTimer = Optional.fromNullable( delayTimer );
+ this.timeUnitOptional = Optional.fromNullable( timeUnit );
+
+ return this;
+ }
+
+
+ /**
+ * Set end timestamp in epoch time. Only entities created before this time will be processed for deletion
+ * @param timestamp
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withEndTimestamp( final Long timestamp ) {
+ this.endTimestamp = Optional.fromNullable( timestamp );
+ return this;
+ }
+
+
+ @Override
+ public Optional<Integer> getDelayTimer() {
+ return delayTimer;
+ }
+
+ @Override
+ public Optional<TimeUnit> getTimeUnitOptional() {
+ return timeUnitOptional;
+ }
+
+
+ @Override
+ public Optional<ApplicationScope> getApplicationScope() {
+
+ if ( this.withApplicationId.isPresent() ) {
+ return Optional.of( CpNamingUtils.getApplicationScope( withApplicationId.get() ) );
+ }
+
+ return Optional.absent();
+ }
+
+
+ @Override
+ public Optional<String> getCollectionName() {
+ return withCollectionName;
+ }
+
+
+ @Override
+ public Optional<String> getCursor() {
+ return cursor;
+ }
+
+
+ @Override
+ public Optional<Long> getEndTimestamp() {
+ return endTimestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteService.java
new file mode 100644
index 0000000..c939dd3
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteService.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+/**
+ * An interface for re-indexing all entities in an application
+ */
+public interface CollectionDeleteService {
+
+
+ /**
+ * Perform a collection delete via service
+ *
+ * @param collectionDeleteRequestBuilder The builder to build the request
+ */
+ CollectionDeleteStatus deleteCollection(final CollectionDeleteRequestBuilder collectionDeleteRequestBuilder);
+
+
+ /**
+ * Generate a build for the collection delete
+ */
+ CollectionDeleteRequestBuilder getBuilder();
+
+
+ /**
+ * Get the status of a job
+ * @param jobId The jobId returned during the collection delete
+ * @return
+ */
+ CollectionDeleteStatus getStatus(final String jobId);
+
+
+ /**
+ * The response when requesting a collection delete operation
+ */
+ public class CollectionDeleteStatus {
+ final String jobId;
+ final Status status;
+ final long numberProcessed;
+ final long lastUpdated;
+
+
+ public CollectionDeleteStatus(final String jobId, final Status status, final long numberProcessed,
+ final long lastUpdated ) {
+ this.jobId = jobId;
+ this.status = status;
+ this.numberProcessed = numberProcessed;
+ this.lastUpdated = lastUpdated;
+ }
+
+
+ /**
+ * Get the jobId used to resume this operation
+ */
+ public String getJobId() {
+ return jobId;
+ }
+
+
+ /**
+ * Get the last updated time, as a long
+ * @return
+ */
+ public long getLastUpdated() {
+ return lastUpdated;
+ }
+
+
+ /**
+ * Get the number of records processed
+ * @return
+ */
+ public long getNumberProcessed() {
+ return numberProcessed;
+ }
+
+
+ /**
+ * Get the status
+ * @return
+ */
+ public Status getStatus() {
+ return status;
+ }
+ }
+
+ enum Status{
+ STARTED, INPROGRESS, COMPLETE, UNKNOWN;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteServiceImpl.java
new file mode 100644
index 0000000..7b3e324
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteServiceImpl.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializerUtil;
+import org.apache.usergrid.corepersistence.pipeline.read.CursorSeek;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.StringUtils;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.apache.usergrid.utils.JsonUtils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
+import static com.google.common.base.Optional.fromNullable;
+
+
+@Singleton
+public class CollectionDeleteServiceImpl implements CollectionDeleteService {
+
+ private static final Logger logger = LoggerFactory.getLogger( CollectionDeleteServiceImpl.class );
+
+ private static final MapScope RESUME_MAP_SCOPE =
+ new MapScopeImpl( CpNamingUtils.getManagementApplicationId(), "collectiondeleteresume" );
+
+ //Keep cursors to resume collection delete for 10 days.
+ private static final int CURSOR_TTL = 60 * 60 * 24 * 10;
+
+ private static final String MAP_CURSOR_KEY = "cursor";
+ private static final String MAP_COUNT_KEY = "count";
+ private static final String MAP_STATUS_KEY = "status";
+ private static final String MAP_UPDATED_KEY = "lastUpdated";
+
+
+ private final AllApplicationsObservable allApplicationsObservable;
+ private final IndexLocationStrategyFactory indexLocationStrategyFactory;
+ private final AllEntityIdsObservable allEntityIdsObservable;
+ private final IndexProcessorFig indexProcessorFig;
+ private final MapManager mapManager;
+ private final MapManagerFactory mapManagerFactory;
+ private final AsyncEventService indexService;
+ private final EntityIndexFactory entityIndexFactory;
+ private final CollectionSettingsFactory collectionSettingsFactory;
+
+
+ @Inject
+ public CollectionDeleteServiceImpl(final EntityIndexFactory entityIndexFactory,
+ final IndexLocationStrategyFactory indexLocationStrategyFactory,
+ final AllEntityIdsObservable allEntityIdsObservable,
+ final MapManagerFactory mapManagerFactory,
+ final AllApplicationsObservable allApplicationsObservable,
+ final IndexProcessorFig indexProcessorFig,
+ final CollectionSettingsFactory collectionSettingsFactory,
+ final AsyncEventService indexService ) {
+ this.entityIndexFactory = entityIndexFactory;
+ this.indexLocationStrategyFactory = indexLocationStrategyFactory;
+ this.allEntityIdsObservable = allEntityIdsObservable;
+ this.allApplicationsObservable = allApplicationsObservable;
+ this.indexProcessorFig = indexProcessorFig;
+ this.indexService = indexService;
+ this.collectionSettingsFactory = collectionSettingsFactory;
+ this.mapManagerFactory = mapManagerFactory;
+ this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPE );
+ }
+
+
+ //TODO: optional delay, param.
+ @Override
+ public CollectionDeleteStatus deleteCollection( final CollectionDeleteRequestBuilder collectionDeleteRequestBuilder) {
+
+ final AtomicInteger count = new AtomicInteger();
+
+ final Optional<EdgeScope> cursor = parseCursor( collectionDeleteRequestBuilder.getCursor() );
+
+ final CursorSeek<Edge> cursorSeek = getResumeEdge( cursor );
+
+ final Optional<Integer> delayTimer = collectionDeleteRequestBuilder.getDelayTimer();
+
+ final Optional<TimeUnit> timeUnitOptional = collectionDeleteRequestBuilder.getTimeUnitOptional();
+
+ Optional<ApplicationScope> appId = collectionDeleteRequestBuilder.getApplicationScope();
+
+ Preconditions.checkArgument(collectionDeleteRequestBuilder.getCollectionName().isPresent(),
+ "You must specify a collection name");
+ String collectionName = collectionDeleteRequestBuilder.getCollectionName().get();
+
+ Preconditions.checkArgument( !(cursor.isPresent() && appId.isPresent()),
+ "You cannot specify an app id and a cursor. When resuming with cursor you must omit the appid." );
+ Preconditions.checkArgument( cursor.isPresent() || appId.isPresent(),
+ "Either application ID or cursor is required.");
+
+ ApplicationScope applicationScope;
+ if (appId.isPresent()) {
+ applicationScope = appId.get();
+ } else { // cursor is present
+ applicationScope = cursor.get().getApplicationScope();
+ }
+
+
+ final String jobId = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+
+ // default to current time
+ final long endTimestamp = collectionDeleteRequestBuilder.getEndTimestamp().or( System.currentTimeMillis() );
+
+ String pluralizedCollectionName = InflectionUtils.pluralize(CpNamingUtils.getNameFromEdgeType(collectionName));
+
+ CollectionSettings collectionSettings =
+ collectionSettingsFactory.getInstance(new CollectionSettingsScopeImpl(applicationScope.getApplication(), pluralizedCollectionName));
+
+ Optional<Map<String, Object>> existingSettings =
+ collectionSettings.getCollectionSettings( pluralizedCollectionName );
+
+ if ( existingSettings.isPresent() ) {
+
+ Map jsonMapData = existingSettings.get();
+
+ jsonMapData.put( "lastCollectionClear", Instant.now().toEpochMilli() );
+
+ collectionSettings.putCollectionSettings(
+ pluralizedCollectionName, JsonUtils.mapToJsonString(jsonMapData ) );
+ }
+
+ allEntityIdsObservable.getEdgesToEntities( Observable.just(applicationScope),
+ fromNullable(collectionName), cursorSeek.getSeekValue() )
+ .buffer( indexProcessorFig.getCollectionDeleteBufferSize())
+ .doOnNext( edgeScopes -> {
+ logger.info("Sending batch of {} to be deleted.", edgeScopes.size());
+ indexService.deleteBatch(edgeScopes, endTimestamp, AsyncEventQueueType.DELETE);
+ count.addAndGet(edgeScopes.size() );
+ if( edgeScopes.size() > 0 ) {
+ writeCursorState(jobId, edgeScopes.get(edgeScopes.size() - 1));
+ }
+ writeStateMeta( jobId, Status.INPROGRESS, count.get(), System.currentTimeMillis() ); })
+ .doOnCompleted(() -> writeStateMeta( jobId, Status.COMPLETE, count.get(), System.currentTimeMillis() ))
+ .subscribeOn( Schedulers.io() ).subscribe();
+
+
+ return new CollectionDeleteStatus( jobId, Status.STARTED, 0, 0 );
+ }
+
+
+ @Override
+ public CollectionDeleteRequestBuilder getBuilder() {
+ return new CollectionDeleteRequestBuilderImpl();
+ }
+
+
+ @Override
+ public CollectionDeleteStatus getStatus( final String jobId ) {
+ Preconditions.checkNotNull( jobId, "jobId must not be null" );
+ return getCollectionDeleteResponse( jobId );
+ }
+
+
+ /**
+ * Get the resume edge scope
+ *
+ * @param edgeScope The optional edge scope from the cursor
+ */
+ private CursorSeek<Edge> getResumeEdge( final Optional<EdgeScope> edgeScope ) {
+
+
+ if ( edgeScope.isPresent() ) {
+ return new CursorSeek<>( Optional.of( edgeScope.get().getEdge() ) );
+ }
+
+ return new CursorSeek<>( Optional.absent() );
+ }
+
+
+ /**
+ * Swap our cursor for an optional edgescope
+ */
+ private Optional<EdgeScope> parseCursor( final Optional<String> cursor ) {
+
+ if ( !cursor.isPresent() ) {
+ return Optional.absent();
+ }
+
+ //get our cursor
+ final String persistedCursor = mapManager.getString( cursor.get() );
+
+ if ( persistedCursor == null ) {
+ return Optional.absent();
+ }
+
+ final JsonNode node = CursorSerializerUtil.fromString( persistedCursor );
+
+ final EdgeScope edgeScope = EdgeScopeSerializer.INSTANCE.fromJsonNode( node, CursorSerializerUtil.getMapper() );
+
+ return Optional.of( edgeScope );
+ }
+
+
+ /**
+ * Write the cursor state to the map in cassandra
+ */
+ private void writeCursorState( final String jobId, final EdgeScope edge ) {
+
+ final JsonNode node = EdgeScopeSerializer.INSTANCE.toNode( CursorSerializerUtil.getMapper(), edge );
+
+ final String serializedState = CursorSerializerUtil.asString( node );
+
+ mapManager.putString( jobId + MAP_CURSOR_KEY, serializedState, CURSOR_TTL);
+ }
+
+
+ /**
+ * Write our state meta data into cassandra so everyone can see it
+ * @param jobId
+ * @param status
+ * @param processedCount
+ * @param lastUpdated
+ */
+ private void writeStateMeta( final String jobId, final Status status, final long processedCount,
+ final long lastUpdated ) {
+
+ if(logger.isDebugEnabled()) {
+ logger.debug( "Flushing state for jobId {}, status {}, processedCount {}, lastUpdated {}",
+ jobId, status, processedCount, lastUpdated);
+ }
+
+ mapManager.putString( jobId + MAP_STATUS_KEY, status.name() );
+ mapManager.putLong( jobId + MAP_COUNT_KEY, processedCount );
+ mapManager.putLong( jobId + MAP_UPDATED_KEY, lastUpdated );
+ }
+
+
+ /**
+ * Get the index response from the jobId
+ * @param jobId
+ * @return
+ */
+ private CollectionDeleteStatus getCollectionDeleteResponse( final String jobId ) {
+
+ final String stringStatus = mapManager.getString( jobId+MAP_STATUS_KEY );
+
+ if(stringStatus == null){
+ return new CollectionDeleteStatus( jobId, Status.UNKNOWN, 0, 0 );
+ }
+
+ final Status status = Status.valueOf( stringStatus );
+
+ final long processedCount = mapManager.getLong( jobId + MAP_COUNT_KEY );
+ final long lastUpdated = mapManager.getLong( jobId + MAP_UPDATED_KEY );
+
+ return new CollectionDeleteStatus( jobId, status, processedCount, lastUpdated );
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index eb63056..948e106 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -56,6 +56,8 @@ public interface IndexProcessorFig extends GuicyFig {
String REINDEX_BUFFER_SIZE = "elasticsearch.reindex.buffer_size";
+ String COLLECTION_DELETE_BUFFER_SIZE = "elasticsearch.collection_delete.buffer_size";
+
String REINDEX_CONCURRENCY_FACTOR = "elasticsearch.reindex.concurrency.factor";
@@ -157,6 +159,13 @@ public interface IndexProcessorFig extends GuicyFig {
int getReindexConcurrencyFactor();
/**
+ * Number of parallel buffers during collection delete
+ */
+ @Default("500")
+ @Key(COLLECTION_DELETE_BUFFER_SIZE)
+ int getCollectionDeleteBufferSize();
+
+ /**
* Flag to resolve the LOCAL queue implementation service synchronously.
*/
@Default("false")
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index c7371b3..05602fc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -352,7 +352,7 @@ public class ReIndexServiceImpl implements ReIndexService {
final Status status = Status.valueOf( stringStatus );
final long processedCount = mapManager.getLong( jobId + MAP_COUNT_KEY );
- final long lastUpdated = mapManager.getLong( jobId + MAP_COUNT_KEY );
+ final long lastUpdated = mapManager.getLong( jobId + MAP_UPDATED_KEY );
return new ReIndexStatus( jobId, status, processedCount, lastUpdated );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
new file mode 100644
index 0000000..ddf2c68
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence;
+
+
+import com.codahale.metrics.MetricRegistry;
+import com.google.inject.Injector;
+import net.jcip.annotations.NotThreadSafe;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.cassandra.SpringResource;
+import org.apache.usergrid.corepersistence.index.*;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+import static org.junit.Assert.*;
+
+
+@NotThreadSafe
+public class CollectionDeleteTest extends AbstractCoreIT {
+ private static final Logger logger = LoggerFactory.getLogger( CollectionDeleteTest.class );
+
+ private static final MetricRegistry registry = new MetricRegistry();
+
+
+ private static final int ENTITIES_TO_DELETE = 1000;
+ private static final int ENTITIES_TO_ADD_AFTER_TIME = 3;
+
+
+ @Before
+ public void startReporting() {
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Starting metrics reporting");
+ }
+ }
+
+
+ @After
+ public void printReport() {
+ logger.debug( "Printing metrics report" );
+ }
+
+
+ @Test( timeout = 240000 )
+ public void clearOneCollection() throws Exception {
+
+ logger.info( "Started clearOneCollection()" );
+
+ String rand = RandomStringUtils.randomAlphanumeric( 5 );
+ final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
+
+ final EntityManager em = setup.getEmf().getEntityManager( appId );
+
+ final CollectionDeleteService collectionDeleteService = setup.getInjector().getInstance( CollectionDeleteService.class );
+
+ // ----------------- create a bunch of entities
+
+ Map<String, Object> entityMap = new HashMap<String, Object>() {{
+ put( "key1", 1000 );
+ put( "key2", 2000 );
+ put( "key3", "Some value" );
+ }};
+
+ String collectionName = "items";
+ String itemType = "item";
+
+
+ List<EntityRef> entityRefs = new ArrayList<EntityRef>();
+ for ( int i = 0; i < ENTITIES_TO_DELETE; i++ ) {
+
+ final Entity entity;
+
+ try {
+ entityMap.put( "key", i );
+ entity = em.create(itemType, entityMap);
+ }
+ catch ( Exception ex ) {
+ throw new RuntimeException( "Error creating entity", ex );
+ }
+
+ entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
+ if ( i % 10 == 0 ) {
+ logger.info( "Created {} entities", i );
+ }
+ }
+
+ logger.info("Created {} entities", ENTITIES_TO_DELETE);
+ long timeFirstPutDone = System.currentTimeMillis();
+ logger.info("timeFirstPutDone={}", timeFirstPutDone);
+
+ for (int i = 0; i < ENTITIES_TO_ADD_AFTER_TIME; i++) {
+
+ final Entity entity;
+
+ try {
+ entityMap.put( "key", ENTITIES_TO_DELETE + i );
+ entity = em.create(itemType, entityMap);
+ }
+ catch ( Exception ex ) {
+ throw new RuntimeException( "Error creating entity", ex );
+ }
+
+ entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
+ if ( i % 10 == 0 ) {
+ logger.info( "Created {} entities after delete time", i );
+ }
+
+ }
+ logger.info("Created {} entities after delete time", ENTITIES_TO_ADD_AFTER_TIME);
+
+
+ app.waitForQueueDrainAndRefreshIndex(5000);
+
+ final CollectionDeleteRequestBuilder builder =
+ collectionDeleteService.getBuilder()
+ .withApplicationId( em.getApplicationId() )
+ .withCollection(collectionName)
+ .withEndTimestamp(timeFirstPutDone);
+
+ CollectionDeleteService.CollectionDeleteStatus status = collectionDeleteService.deleteCollection(builder);
+
+ assertNotNull( status.getJobId(), "JobId is present" );
+
+ logger.info( "Delete collection" );
+
+
+ waitForDelete( status, collectionDeleteService );
+
+ app.waitForQueueDrainAndRefreshIndex(15000);
+
+ // ----------------- test that we can read the entries after the timestamp
+
+ readData( em, collectionName,ENTITIES_TO_ADD_AFTER_TIME);
+ }
+
+ /**
+ * Wait for the delete to occur
+ */
+ private void waitForDelete( final CollectionDeleteService.CollectionDeleteStatus status, final CollectionDeleteService collectionDeleteService )
+ throws InterruptedException, IllegalArgumentException {
+ if (status != null) {
+ logger.info("waitForDelete: jobID={}", status.getJobId());
+ } else {
+ logger.info("waitForDelete: error, status = null");
+ throw new IllegalArgumentException("collectionDeleteStatus = null");
+ }
+ while ( true ) {
+
+ try {
+ final CollectionDeleteService.CollectionDeleteStatus updatedStatus =
+ collectionDeleteService.getStatus( status.getJobId() );
+
+ if (updatedStatus == null) {
+ logger.info("waitForDelete: updated status is null");
+ } else {
+ logger.info("waitForDelete: status={} numberProcessed={}",
+ updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed());
+
+ if ( updatedStatus.getStatus() == CollectionDeleteService.Status.COMPLETE ) {
+ break;
+ }
+ }
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow. Thrown if our job can't be found. I.E hasn't updated yet
+ }
+
+
+ Thread.sleep( 1000 );
+ }
+ }
+
+
+ private int readData(EntityManager em, String collectionName, int expectedEntities)
+ throws Exception {
+
+ app.waitForQueueDrainAndRefreshIndex();
+
+ Results results = em.getCollection(em.getApplicationRef(), collectionName, null, expectedEntities,
+ Query.Level.ALL_PROPERTIES, false);
+
+ int count = 0;
+ while ( true ) {
+
+ if (results.getEntities().size() == 0) {
+ break;
+ }
+
+ UUID lastEntityUUID = null;
+ for ( Entity e : results.getEntities() ) {
+
+ assertEquals(2000, e.getProperty("key2"));
+
+ if (count % 100 == 0) {
+ logger.info("read {} entities", count);
+ }
+ lastEntityUUID = e.getUuid();
+ count++;
+ }
+
+ results = em.getCollection(em.getApplicationRef(), collectionName, lastEntityUUID, expectedEntities,
+ Query.Level.ALL_PROPERTIES, false);
+
+ }
+ logger.info("read {} total entities", count);
+
+ assertEquals( "Did not get expected entities", expectedEntities, count );
+ return count;
+ }
+
+ private int countEntities( EntityManager em, String collectionName, int expectedEntities)
+ throws Exception {
+
+ app.waitForQueueDrainAndRefreshIndex();
+
+ Query q = Query.fromQL( "select * where key1=1000" ).withLimit( 1000 );
+ Results results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, expectedEntities );
+
+ int count = 0;
+ while ( true ) {
+
+ count += results.size();
+
+
+ if ( results.hasCursor() ) {
+ logger.info( "Counted {} : query again with cursor", count );
+ q.setCursor( results.getCursor() );
+ results = em.searchCollection( em.getApplicationRef(), collectionName, q );
+ }
+ else {
+ break;
+ }
+ }
+
+ assertEquals( "Did not get expected entities", expectedEntities, count );
+ return count;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/query-validator/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/query-validator/src/test/resources/usergrid-custom-test.properties b/stack/query-validator/src/test/resources/usergrid-custom-test.properties
index bc1ba56..c8e3eee 100644
--- a/stack/query-validator/src/test/resources/usergrid-custom-test.properties
+++ b/stack/query-validator/src/test/resources/usergrid-custom-test.properties
@@ -30,3 +30,5 @@ usergrid.sysadmin.login.allowed=true
# This property is required to be set and cannot be defaulted anywhere
usergrid.cluster_name=usergrid
+
+elasticsearch.queue_impl=LOCAL
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
index b8c1caa..c9174c1 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
@@ -18,19 +18,19 @@
package org.apache.usergrid.rest.applications;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.PathSegment;
import javax.ws.rs.core.UriInfo;
+import com.google.common.base.Preconditions;
+import org.apache.usergrid.corepersistence.index.CollectionDeleteRequestBuilder;
+import org.apache.usergrid.corepersistence.index.CollectionDeleteRequestBuilderImpl;
+import org.apache.usergrid.corepersistence.index.CollectionDeleteService;
+import org.apache.usergrid.persistence.index.utils.ConversionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
@@ -48,6 +48,9 @@ import org.apache.usergrid.services.ServicePayload;
import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+import java.util.HashMap;
+import java.util.Map;
+
/**
* A collection resource that stands before the Service Resource. If it cannot find
@@ -61,6 +64,9 @@ import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
})
public class CollectionResource extends ServiceResource {
+ private static final Logger logger = LoggerFactory.getLogger( CollectionResource.class );
+ private static final String UPDATED_BEFORE_FIELD = "updatedBefore";
+
public CollectionResource() {
}
@@ -190,6 +196,61 @@ public class CollectionResource extends ServiceResource {
}
+ @PUT
+ @Path("{itemName}/_clear")
+ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
+ @RequireApplicationAccess
+ @JSONP
+ public ApiResponse clearCollectionPut(
+ final Map<String, Object> payload,
+ @PathParam("itemName") final String collectionName,
+ @QueryParam("callback") @DefaultValue("callback") String callback
+ ) throws Exception {
+
+ logger.info("Clearing collection {} for application {}", collectionName, getApplicationId().toString());
+
+ final CollectionDeleteRequestBuilder request = createRequest()
+ .withApplicationId(getApplicationId())
+ .withCollection(collectionName);
+
+ return executeResumeAndCreateResponse(payload, request, callback);
+
+ }
+
+
+ @GET
+ @Path( "{itemName}/_clear/{jobId}")
+ @Produces({MediaType.APPLICATION_JSON,"application/javascript"})
+ @RequireApplicationAccess
+ @JSONP
+ public ApiResponse clearCollectionJobGet(
+ @Context UriInfo ui,
+ @PathParam("itemName") PathSegment itemName,
+ @PathParam("jobId") String jobId,
+ @QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {
+
+ if(logger.isTraceEnabled()){
+ logger.trace( "CollectionResource.clearCollectionJobGet" );
+ }
+
+ Preconditions
+ .checkNotNull(jobId, "path param jobId must not be null" );
+
+ CollectionDeleteService.CollectionDeleteStatus status = getCollectionDeleteService().getStatus(jobId);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear collection" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberCheckedForDeletion", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return response;
+ }
+
+
// TODO: this can't be controlled and until it can be controlled we shouldn' allow muggles to do this.
// So system access only.
// TODO: use scheduler here to get around people sending a reindex call 30 times.
@@ -210,4 +271,57 @@ public class CollectionResource extends ServiceResource {
services.getApplicationId().toString(),itemName.getPath(),false,callback );
}
+
+ private CollectionDeleteService getCollectionDeleteService() {
+ return injector.getInstance( CollectionDeleteService.class );
+ }
+
+
+ private CollectionDeleteRequestBuilder createRequest() {
+ return new CollectionDeleteRequestBuilderImpl();
+ }
+
+
+ private ApiResponse executeResumeAndCreateResponse( final Map<String, Object> payload,
+ final CollectionDeleteRequestBuilder request,
+ final String callback ) {
+
+ Map<String,Object> newPayload = payload;
+ if(newPayload == null || !payload.containsKey( UPDATED_BEFORE_FIELD )){
+ newPayload = new HashMap<>(1);
+ newPayload.put(UPDATED_BEFORE_FIELD,Long.MAX_VALUE);
+ }
+
+ Preconditions.checkArgument(newPayload.get(UPDATED_BEFORE_FIELD) instanceof Number,
+ "The field \"updatedBefore\" in the payload must be a timestamp" );
+
+ //add our updated timestamp to the request
+ if ( newPayload.containsKey( UPDATED_BEFORE_FIELD ) ) {
+ final long timestamp = ConversionUtils.getLong(newPayload.get(UPDATED_BEFORE_FIELD));
+ request.withEndTimestamp( timestamp );
+ }
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+ /**
+ * Execute the request and return the response.
+ */
+ private ApiResponse executeAndCreateResponse(final CollectionDeleteRequestBuilder request, final String callback ) {
+
+
+ final CollectionDeleteService.CollectionDeleteStatus status = getCollectionDeleteService().deleteCollection( request );
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear collection" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return response;
+ }
+
}
[09/15] usergrid git commit: test changes and additional logging
Posted by md...@apache.org.
test changes and additional logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/39ec4f2b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/39ec4f2b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/39ec4f2b
Branch: refs/heads/master
Commit: 39ec4f2bb895e180f493349fb14e62ac9178e5ee
Parents: 400365c
Author: Mike Dunker <md...@google.com>
Authored: Fri Aug 18 09:46:50 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Fri Aug 18 09:46:50 2017 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../asyncevents/AsyncEventServiceImpl.java | 2 ++
.../apache/usergrid/persistence/RebuildIndexTest.java | 12 ++++++------
.../queues/impl/QueueSerializationImpl.java | 2 +-
.../apache/usergrid/services/ActivitiesServiceIT.java | 9 ++++++---
.../apache/usergrid/services/ServiceInvocationIT.java | 3 +++
6 files changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index cec7258..bad5b2c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -487,7 +487,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// evict app Id from cache
applicationIdCache.evictAppId(appName);
- logger.info("Initialized application {}", appName);
+ logger.info("Initialized application {}, uuid {}", appName, appInfo.getUuid().toString());
return appInfo;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 530cf7d..9501ad3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -881,6 +881,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final int utilityCount = indexProcessorFig.getWorkerCountUtility();
final int indexDeadCount = indexProcessorFig.getWorkerCountDeadLetter();
final int utilityDeadCount = indexProcessorFig.getWorkerCountUtilityDeadLetter();
+ logger.info("Starting queue workers for indexing: index={} indexDLQ={} utility={} utilityDLQ={}", indexCount, indexDeadCount,
+ utilityCount, utilityDeadCount);
for (int i = 0; i < indexCount; i++) {
startWorker(QUEUE_NAME);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index a7759de..d2bff37 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -77,7 +77,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
@Test( timeout = 120000 )
public void rebuildOneCollectionIndex() throws Exception {
- logger.info( "Started rebuildIndex()" );
+ logger.info( "Started rebuildOneCollectionIndex()" );
String rand = RandomStringUtils.randomAlphanumeric( 5 );
final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
@@ -172,7 +172,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
}
- @Test( timeout = 120000 )
+ @Test( timeout = 240000 )
public void rebuildIndex() throws Exception {
logger.info( "Started rebuildIndex()" );
@@ -234,7 +234,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
}
logger.info( "Created {} entities", ENTITIES_TO_INDEX );
- app.waitForQueueDrainAndRefreshIndex(15000);
+ app.waitForQueueDrainAndRefreshIndex(30000);
// ----------------- test that we can read them, should work fine
@@ -301,7 +301,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
@Test( timeout = 120000 )
public void rebuildIndexGeo() throws Exception {
- logger.info( "Started rebuildIndex()" );
+ logger.info( "Started rebuildIndexGeo()" );
String rand = RandomStringUtils.randomAlphanumeric( 5 );
final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
@@ -414,7 +414,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
@Test( timeout = 120000 )
public void rebuildUpdatedSince() throws Exception {
- logger.info( "Started rebuildIndex()" );
+ logger.info( "Started rebuildUpdatedSince()" );
String rand = RandomStringUtils.randomAlphanumeric( 5 );
final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
@@ -436,7 +436,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
final Entity secondEntity = em.create( "thing", entityData);
- app.waitForQueueDrainAndRefreshIndex(5000);
+ app.waitForQueueDrainAndRefreshIndex(15000);
// ----------------- test that we can read them, should work fine
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java
index d3a46aa..2ed37e0 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java
@@ -163,7 +163,7 @@ public class QueueSerializationImpl implements QueueSerialization {
@Override
public Collection<TableDefinition> getTables() {
return Collections.singletonList(
- new TableDefinitionStringImpl( cassandraConfig.getApplicationKeyspace(), "queues", CQL ) );
+ new TableDefinitionStringImpl( cassandraConfig.getApplicationKeyspace(), TABLE_QUEUES, CQL ) );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java
index c4762f2..c8a446a 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java
@@ -93,11 +93,14 @@ public class ActivitiesServiceIT extends AbstractServiceIT {
app.testRequest( ServiceAction.GET, 4, null, "users", userC.getUuid(), "feed" );
- app.testRequest( ServiceAction.GET, 2, null, "users", userC.getUuid(), "feed",
- Query.fromQL( "select * where content contains 'cookie'" ) );
+ // time for indexing
+ Thread.sleep(10000);
app.testRequest( ServiceAction.GET, 1, "users", userC.getUuid(), "feed",
- Query.fromQL( "select * where verb='post' and content contains 'cookie'" ) );
+ Query.fromQL( "select * where verb='post' and content contains 'cookie'" ) );
+
+ app.testRequest( ServiceAction.GET, 2, null, "users", userC.getUuid(), "feed",
+ Query.fromQL( "select * where content contains 'cookie'" ) );
app.put( "username", "finn" );
app.put( "email", "finn@ooo.com" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
index 81dced1..434fd80 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
@@ -102,6 +102,9 @@ public class ServiceInvocationIT extends AbstractServiceIT {
app.testRequest( ServiceAction.GET, 2, "users", "edanuff", "likes", "restaurants" );
+ // time for indexing
+ Thread.sleep(10000);
+
app.testRequest( ServiceAction.GET, 1, "users", "edanuff", "likes", "restaurants",
Query.fromQL( "select * where name='Brickhouse'" ) );
[03/15] usergrid git commit: fix broken recaptcha
Posted by md...@apache.org.
fix broken recaptcha
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/459163b8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/459163b8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/459163b8
Branch: refs/heads/master
Commit: 459163b8ddb4b80ebb43273a8646ee9c8a6fe2bd
Parents: be2f69c
Author: Mike Dunker <md...@google.com>
Authored: Wed Aug 9 18:39:23 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 09:47:40 2017 -0700
----------------------------------------------------------------------
.../rest/applications/ApplicationResource/authorize_form.jsp | 2 +-
.../rest/applications/users/UserResource/resetpw_email_form.jsp | 4 ++--
.../rest/applications/users/UserResource/resetpw_set_form.jsp | 2 +-
.../rest/applications/users/UsersResource/resetpw_email_form.jsp | 4 ++--
.../rest/management/ManagementResource/authorize_form.jsp | 2 +-
.../rest/management/users/UserResource/resetpw_email_form.jsp | 4 ++--
.../rest/management/users/UserResource/resetpw_set_form.jsp | 2 +-
.../rest/management/users/UsersResource/resetpw_email_form.jsp | 4 ++--
8 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
index 0079bcf..ed934a7 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
@@ -29,7 +29,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<input type="hidden" name="response_type" value="${fn:escapeXml(it.responseType)}">
<input type="hidden" name="client_id" value="${fn:escapeXml(it.clientId)}">
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
index 59026bf..5230ea7 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <c:if test="${!empty it.errorMsg}">
<div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
@@ -37,7 +37,7 @@ limitations under the License.
<c:out value="${it.user.email}" escapeXml="true" />
</p>
<p id="human-proof"></p>
- ${fn:escapeXml(it.reCaptchaHtml)}
+ ${it.reCaptchaHtml}
<p class="buttons">
<input type="submit" value="submit" />
</p>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
index 19b6528..c0203ce 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
@@ -31,7 +31,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<input type="hidden" name="token" value="${fn:escapeXml(it.token)}">
<fieldset>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
index f86240f..01dfc57 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
@@ -28,7 +28,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <c:if test="${!empty it.errorMsg}">
<div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
@@ -42,7 +42,7 @@ limitations under the License.
<input class="text_field" id="email" name="email" type="text" />
</p>
<p id="human-proof"></p>
- ${fn:escapeXml(it.reCaptchaHtml)}
+ ${it.reCaptchaHtml}
<p class="buttons">
<button type="submit">Submit</button>
</p>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
index 0079bcf..ed934a7 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
@@ -29,7 +29,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<input type="hidden" name="response_type" value="${fn:escapeXml(it.responseType)}">
<input type="hidden" name="client_id" value="${fn:escapeXml(it.clientId)}">
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
index c9f8309..6341d60 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
<div class="dialog-area password-reset-form">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <c:if test="${!empty it.errorMsg}">
<div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
@@ -37,7 +37,7 @@ limitations under the License.
<c:out value="${it.user.email}" escapeXml="true" />
</p>
<p id="human-proof"></p>
- ${fn:escapeXml(it.reCaptchaHtml)}
+ ${it.reCaptchaHtml}
<p class="buttons">
<input type="submit" value="submit" />
</p>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
index 60384c4..6334466 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
@@ -30,7 +30,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<input type="hidden" name="token" value="${fn:escapeXml(it.token)}">
<fieldset>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
index 8b15cd4..f8cf496 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
@@ -27,7 +27,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escnapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
<p>
@@ -38,7 +38,7 @@ limitations under the License.
<input class="text_field" id="email" name="email" type="text" />
</p>
<p id="human-proof"></p>
- ${fn:escapeXml(it.reCaptchaHtml)}
+ ${it.reCaptchaHtml}
<p class="buttons">
<button type="submit">Submit</button>
</p>
[02/15] usergrid git commit: fix html encode in jsp
Posted by md...@apache.org.
fix html encode in jsp
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/be2f69c3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/be2f69c3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/be2f69c3
Branch: refs/heads/master
Commit: be2f69c37cab7214382d99e36e27347fc9f3b09b
Parents: 068263e
Author: Mike Dunker <md...@google.com>
Authored: Fri Jul 28 14:11:31 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 09:47:00 2017 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/rest/TestResource/error.jsp | 4 ++--
.../org/apache/usergrid/rest/TestResource/test.jsp | 5 +++--
.../ApplicationResource/authorize_form.jsp | 15 ++++++++-------
.../rest/applications/ApplicationResource/error.jsp | 4 ++--
.../applications/users/UserResource/activate.jsp | 4 ++--
.../rest/applications/users/UserResource/confirm.jsp | 4 ++--
.../rest/applications/users/UserResource/error.jsp | 4 ++--
.../users/UserResource/resetpw_email_form.jsp | 11 ++++++-----
.../users/UserResource/resetpw_email_success.jsp | 4 ++--
.../users/UserResource/resetpw_set_form.jsp | 9 +++++----
.../users/UserResource/resetpw_set_success.jsp | 4 ++--
.../rest/applications/users/UsersResource/error.jsp | 4 ++--
.../users/UsersResource/resetpw_email_form.jsp | 9 +++++----
.../users/UsersResource/resetpw_email_success.jsp | 4 ++--
.../management/ManagementResource/authorize_form.jsp | 15 ++++++++-------
.../rest/management/ManagementResource/error.jsp | 4 ++--
.../organizations/OrganizationResource/activate.jsp | 4 ++--
.../organizations/OrganizationResource/confirm.jsp | 4 ++--
.../organizations/OrganizationResource/error.jsp | 4 ++--
.../rest/management/users/UserResource/activate.jsp | 4 ++--
.../rest/management/users/UserResource/confirm.jsp | 4 ++--
.../rest/management/users/UserResource/error.jsp | 4 ++--
.../users/UserResource/resetpw_email_form.jsp | 9 +++++----
.../users/UserResource/resetpw_email_success.jsp | 4 ++--
.../users/UserResource/resetpw_set_form.jsp | 8 ++++----
.../users/UserResource/resetpw_set_success.jsp | 4 ++--
.../rest/management/users/UsersResource/error.jsp | 4 ++--
.../users/UsersResource/resetpw_email_form.jsp | 6 +++---
.../users/UsersResource/resetpw_email_success.jsp | 4 ++--
29 files changed, 87 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
index 83a6ad1..68c12f2 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
@@ -1,5 +1,6 @@
<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -24,6 +25,6 @@ limitations under the License.
<link rel="stylesheet" type="text/css" href="/css/styles.css" />
</head>
<body>
-<h1>${it.foo}</h1>
+<h1>${fn:escapeXml(it.foo)}</h1>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
index 6b1b8b2..0079bcf 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
@@ -2,6 +2,7 @@
pageEncoding="ISO-8859-1"%>
<%@ page import="org.apache.usergrid.rest.AbstractContextResource"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -28,13 +29,13 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
- <input type="hidden" name="response_type" value="${it.responseType}">
- <input type="hidden" name="client_id" value="${it.clientId}">
- <input type="hidden" name="redirect_uri" value="${it.redirectUri}">
- <input type="hidden" name="scope" value="${it.scope}">
- <input type="hidden" name="state" value="${it.state}">
+ <input type="hidden" name="response_type" value="${fn:escapeXml(it.responseType)}">
+ <input type="hidden" name="client_id" value="${fn:escapeXml(it.clientId)}">
+ <input type="hidden" name="redirect_uri" value="${fn:escapeXml(it.redirectUri)}">
+ <input type="hidden" name="scope" value="${fn:escapeXml(it.scope)}">
+ <input type="hidden" name="state" value="${fn:escapeXml(it.state)}">
<fieldset>
<p>
<label for="username">Username</label>
@@ -56,4 +57,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp
index dfcf3b7..20e69b8 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp
@@ -26,7 +26,7 @@ limitations under the License.
</head>
<body>
- <p>Your account with email address <c:out value="${it.user.email}"/> has been successfully activated.</p>
+ <p>Your account with email address <c:out value="${it.user.email}" escapeXml="true"/> has been successfully activated.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp
index 02e9ee3..d7f3acc 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp
@@ -26,8 +26,8 @@ limitations under the License.
</head>
<body>
- <p>Your account with email address <c:out value="${it.user.email}"/> has been successfully confirmed.
+ <p>Your account with email address <c:out value="${it.user.email}" escapeXml="true"/> has been successfully confirmed.
You will received an email soon to let you know when you account has been activated</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
index 0f53bfc..59026bf 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
@@ -1,6 +1,7 @@
<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -26,17 +27,17 @@ limitations under the License.
</head>
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}">
- <div class="dialog-form-message">${it.errorMsg}</div>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
<p>
Enter the captcha to have your password reset instructions sent to
- <c:out value="${it.user.email}" />
+ <c:out value="${it.user.email}" escapeXml="true" />
</p>
<p id="human-proof"></p>
- ${it.reCaptchaHtml}
+ ${fn:escapeXml(it.reCaptchaHtml)}
<p class="buttons">
<input type="submit" value="submit" />
</p>
@@ -44,4 +45,4 @@ limitations under the License.
</form>
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp
index 23f8508..41c5176 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>Email with instructions for password reset sent to <c:out value="${it.user.email}"/></p>
+ <p>Email with instructions for password reset sent to <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
index a83d80d..19b6528 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
@@ -4,6 +4,7 @@
<%@ page import="net.tanesha.recaptcha.ReCaptchaFactory"%>
<%@ page import="org.apache.usergrid.rest.AbstractContextResource"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -30,12 +31,12 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
- <input type="hidden" name="token" value="${it.token}">
+ <input type="hidden" name="token" value="${fn:escapeXml(it.token)}">
<fieldset>
<p>
- <label for="password1">Please enter your new password for <c:out value="${it.user.email}"/>.</label>
+ <label for="password1">Please enter your new password for <c:out value="${it.user.email}" escapeXml="true"/>.</label>
</p>
<p>
<input class="text_field" id="password1" name="password1" type="password" />
@@ -54,4 +55,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp
index 9de90ba..3915084 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>New password set for <c:out value="${it.user.email}"/></p>
+ <p>New password set for <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
index 3211a3a..f86240f 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
@@ -1,6 +1,7 @@
<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -27,8 +28,8 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}">
- <div class="dialog-form-message">${it.errorMsg}</div>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
@@ -41,7 +42,7 @@ limitations under the License.
<input class="text_field" id="email" name="email" type="text" />
</p>
<p id="human-proof"></p>
- ${it.reCaptchaHtml}
+ ${fn:escapeXml(it.reCaptchaHtml)}
<p class="buttons">
<button type="submit">Submit</button>
</p>
@@ -50,4 +51,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp
index 23f8508..41c5176 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>Email with instructions for password reset sent to <c:out value="${it.user.email}"/></p>
+ <p>Email with instructions for password reset sent to <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
index 6b1b8b2..0079bcf 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
@@ -2,6 +2,7 @@
pageEncoding="ISO-8859-1"%>
<%@ page import="org.apache.usergrid.rest.AbstractContextResource"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -28,13 +29,13 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
- <input type="hidden" name="response_type" value="${it.responseType}">
- <input type="hidden" name="client_id" value="${it.clientId}">
- <input type="hidden" name="redirect_uri" value="${it.redirectUri}">
- <input type="hidden" name="scope" value="${it.scope}">
- <input type="hidden" name="state" value="${it.state}">
+ <input type="hidden" name="response_type" value="${fn:escapeXml(it.responseType)}">
+ <input type="hidden" name="client_id" value="${fn:escapeXml(it.clientId)}">
+ <input type="hidden" name="redirect_uri" value="${fn:escapeXml(it.redirectUri)}">
+ <input type="hidden" name="scope" value="${fn:escapeXml(it.scope)}">
+ <input type="hidden" name="state" value="${fn:escapeXml(it.state)}">
<fieldset>
<p>
<label for="username">Username</label>
@@ -56,4 +57,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp
index 85114cd..f5fa14d 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp
@@ -26,7 +26,7 @@ limitations under the License.
</head>
<body>
- <p>Your organization <c:out value="${it.organization.name}"/> has been successfully activated.</p>
+ <p>Your organization <c:out value="${it.organization.name}" escapeXml="true"/> has been successfully activated.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp
index f4307b7..5fb41c7 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp
@@ -26,8 +26,8 @@ limitations under the License.
</head>
<body>
- <p>Your organization <c:out value="${it.organization.name}"/> has been successfully confirmed.
+ <p>Your organization <c:out value="${it.organization.name}" escapeXml="true"/> has been successfully confirmed.
You will received an email soon to let you know when you organization has been activated</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp
index dfcf3b7..20e69b8 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp
@@ -26,7 +26,7 @@ limitations under the License.
</head>
<body>
- <p>Your account with email address <c:out value="${it.user.email}"/> has been successfully activated.</p>
+ <p>Your account with email address <c:out value="${it.user.email}" escapeXml="true"/> has been successfully activated.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp
index 02e9ee3..d7f3acc 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp
@@ -26,8 +26,8 @@ limitations under the License.
</head>
<body>
- <p>Your account with email address <c:out value="${it.user.email}"/> has been successfully confirmed.
+ <p>Your account with email address <c:out value="${it.user.email}" escapeXml="true"/> has been successfully confirmed.
You will received an email soon to let you know when you account has been activated</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
index 3e56cd1..c9f8309 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
@@ -1,6 +1,7 @@
<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -26,17 +27,17 @@ limitations under the License.
</head>
<body>
<div class="dialog-area password-reset-form">
- <c:if test="${!empty it.errorMsg}">
- <div class="dialog-form-message">${it.errorMsg}</div>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
<p>
Enter the captcha to have your password reset instructions sent to
- <c:out value="${it.user.email}" />
+ <c:out value="${it.user.email}" escapeXml="true" />
</p>
<p id="human-proof"></p>
- ${it.reCaptchaHtml}
+ ${fn:escapeXml(it.reCaptchaHtml)}
<p class="buttons">
<input type="submit" value="submit" />
</p>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp
index 23f8508..41c5176 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>Email with instructions for password reset sent to <c:out value="${it.user.email}"/></p>
+ <p>Email with instructions for password reset sent to <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
index a83d80d..60384c4 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
@@ -30,12 +30,12 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
- <input type="hidden" name="token" value="${it.token}">
+ <input type="hidden" name="token" value="${fn:escapeXml(it.token)}">
<fieldset>
<p>
- <label for="password1">Please enter your new password for <c:out value="${it.user.email}"/>.</label>
+ <label for="password1">Please enter your new password for <c:out value="${it.user.email}" escapeXml="true"/>.</label>
</p>
<p>
<input class="text_field" id="password1" name="password1" type="password" />
@@ -54,4 +54,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp
index 9de90ba..3915084 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>New password set for <c:out value="${it.user.email}"/></p>
+ <p>New password set for <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
index 8643016..8b15cd4 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
@@ -27,7 +27,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escnapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
<p>
@@ -38,7 +38,7 @@ limitations under the License.
<input class="text_field" id="email" name="email" type="text" />
</p>
<p id="human-proof"></p>
- ${it.reCaptchaHtml}
+ ${fn:escapeXml(it.reCaptchaHtml)}
<p class="buttons">
<button type="submit">Submit</button>
</p>
@@ -47,4 +47,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp
index 23f8508..41c5176 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>Email with instructions for password reset sent to <c:out value="${it.user.email}"/></p>
+ <p>Email with instructions for password reset sent to <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
[11/15] usergrid git commit: Converts de-indexing of edges and
entities to identify documents in the index to delete based on data in
Cassandra vs. resource intensive queries to the index. Fixes issue where
nodes were not actually getting deleted from gr
Posted by md...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/websocket/pom.xml
----------------------------------------------------------------------
diff --git a/stack/websocket/pom.xml b/stack/websocket/pom.xml
index af5ed56..72a5c9d 100644
--- a/stack/websocket/pom.xml
+++ b/stack/websocket/pom.xml
@@ -70,7 +70,7 @@
<storage-config>${basedir}/src/test/conf</storage-config>
</systemPropertyVariables>
<forkMode>always</forkMode>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.0/jamm-0.3.0.jar ${ug.argline}</argLine>
</configuration>
</plugin>
</plugins>
[04/15] usergrid git commit: add password complexity check before
submitting during reset password flow
Posted by md...@apache.org.
add password complexity check before submitting during reset password flow
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/70de6fde
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/70de6fde
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/70de6fde
Branch: refs/heads/master
Commit: 70de6fde4438bd4ba750806e5b4a23cd9aa07d9b
Parents: 459163b
Author: Mike Dunker <md...@google.com>
Authored: Thu Aug 17 11:02:36 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 11:02:36 2017 -0700
----------------------------------------------------------------------
.../usergrid/rest/applications/users/UserResource.java | 9 +++++++++
.../usergrid/rest/management/users/UserResource.java | 9 +++++++++
.../org/apache/usergrid/management/ManagementService.java | 10 +++++-----
.../management/cassandra/ManagementServiceImpl.java | 10 ++++++++++
4 files changed, 33 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index 5435f7e..3e4542d 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -17,6 +17,7 @@
package org.apache.usergrid.rest.applications.users;
+import java.util.Collection;
import java.util.Map;
import java.util.UUID;
@@ -465,6 +466,14 @@ public class UserResource extends ServiceResource {
if ( ( password1 != null ) || ( password2 != null ) ) {
if ( management.checkPasswordResetTokenForAppUser( getApplicationId(), getUserUuid(), token ) ) {
if ( ( password1 != null ) && password1.equals( password2 ) ) {
+ // validate password
+ Collection<String> violations = management.passwordPolicyCheck(password1, false);
+ if (violations.size() > 0) {
+ // password not valid
+ errorMsg = management.getPasswordDescription(false);
+ return handleViewable("resetpw_set_form", this, getOrganizationName());
+ }
+
management.setAppUserPassword( getApplicationId(), getUser().getUuid(), password1 );
management.revokeAccessTokenForAppUser( token );
return handleViewable( "resetpw_set_success", this, getOrganizationName() );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index b747aa4..1f80bc1 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -43,6 +43,7 @@ import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriInfo;
+import java.util.Collection;
import java.util.Map;
import java.util.UUID;
@@ -297,6 +298,14 @@ public class UserResource extends AbstractContextResource {
if ( ( password1 != null ) || ( password2 != null ) ) {
if ( management.checkPasswordResetTokenForAdminUser( user.getUuid(), tokenInfo ) ) {
if ( ( password1 != null ) && password1.equals( password2 ) ) {
+ // validate password
+ Collection<String> violations = management.passwordPolicyCheck(password1, true);
+ if (violations.size() > 0) {
+ // password not valid
+ errorMsg = management.getPasswordDescription(true);
+ return handleViewable( "resetpw_set_form", this, organizationId );
+ }
+
management.setAdminUserPassword( user.getUuid(), password1 );
management.revokeAccessTokenForAdminUser( user.getUuid(), token );
loginEndpoint = properties.getProperty("usergrid.viewable.loginEndpoint");
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 2b88b07..8b840d6 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -17,11 +17,7 @@
package org.apache.usergrid.management;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.Entity;
@@ -372,6 +368,10 @@ public interface ManagementService {
Observable<Id> deleteAllEntities(final UUID applicationId,final int limit);
+ Collection<String> passwordPolicyCheck(String password, boolean isAdminUser);
+
+ String getPasswordDescription(boolean isAdminUser);
+
// DO NOT REMOVE BELOW METHODS, THEY ARE HERE TO ALLOW EXTERNAL CLASSES TO OVERRIDE AND HOOK INTO POST PROCESSING
void createOrganizationPostProcessing( final OrganizationInfo orgInfo,
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 2ba9bde..89375fd 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -3412,6 +3412,16 @@ public class ManagementServiceImpl implements ManagementService {
return service.deleteAllEntities(CpNamingUtils.getApplicationScope(applicationId),limit);
}
+ @Override
+ public Collection<String> passwordPolicyCheck(String password, boolean isAdminUser) {
+ return passwordPolicy.policyCheck(password, isAdminUser);
+ }
+
+ @Override
+ public String getPasswordDescription(boolean isAdminUser) {
+ return passwordPolicy.getDescription(isAdminUser);
+ }
+
private String getProperty(String key) {
String obj = properties.getProperty(key);
if(StringUtils.isEmpty(obj))
[06/15] usergrid git commit: change test collection names so no race
condition conflicts
Posted by md...@apache.org.
change test collection names so no race condition conflicts
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d06dbc95
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d06dbc95
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d06dbc95
Branch: refs/heads/master
Commit: d06dbc95a5e41a9cebb20f5859860dd74b6ae9ac
Parents: 4b916c4
Author: Mike Dunker <md...@google.com>
Authored: Wed Aug 16 22:28:54 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 11:59:47 2017 -0700
----------------------------------------------------------------------
.../applications/queries/AndOrQueryTest.java | 111 +++++++--------
.../queries/BadGrammarQueryTest.java | 12 +-
.../rest/applications/queries/OrderByTest.java | 137 ++++++++++---------
.../queries/SelectMappingsQueryTest.java | 53 +++----
4 files changed, 160 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d06dbc95/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
index 4bdd3fc..d49460c 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
@@ -47,20 +47,20 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryAndInclusive() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "apples";
// create our test entities
generateTestEntities(numOfEntities, collectionName);
// Query where madeup = true (the last half) and the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("select * where madeup = true AND ordinal >= " + (numOfEntities - numOfEntities / 4));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
// results should have madeup = true and ordinal 15-19
- assertEquals(numOfEntities / 4, activities.getResponse().getEntityCount());
+ assertEquals(numOfEntities / 4, coll.getResponse().getEntityCount());
// loop though entities that were returned, and test against the ordinals and values we are
// expecting, starting with the last entity and decrementing
int index = 19;
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity activity = coll.next();
// ensure the 'madeup' property is set to true
assertTrue(Boolean.parseBoolean(activity.get("madeup").toString()));
// make sure the correct ordinal properties are returned
@@ -71,15 +71,15 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void someTestProp() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "bananas";
// create our test entities
generateTestEntities(numOfEntities, collectionName);
// Query where madeup = true (the last half) and the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("where sometestprop = 'testprop'");
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
// results should have madeup = true and ordinal 15-19
- assertEquals(10, activities.getResponse().getEntityCount());
+ assertEquals(10, coll.getResponse().getEntityCount());
}
@@ -87,15 +87,15 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void someTestPropPartialContains() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "cantaloupes";
// create our test entities
generateTestEntities(numOfEntities, collectionName);
// Query where madeup = true (the last half) and the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("where sometestprop contains 'test*'");
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
// results should have madeup = true and ordinal 15-19
- assertEquals(10, activities.getResponse().getEntityCount());
+ assertEquals(10, coll.getResponse().getEntityCount());
}
@@ -108,25 +108,25 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryAndExclusive() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "dates";
generateTestEntities(numOfEntities, collectionName);
//Query where madeup = true (the last half) and NOT the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("select * where madeup = true AND NOT ordinal >= " + (numOfEntities - numOfEntities / 4));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//results should have madeup = true and ordinal 10-14
- assertEquals(numOfEntities / 4, activities.getResponse().getEntityCount());
+ assertEquals(numOfEntities / 4, coll.getResponse().getEntityCount());
// loop though entities that were returned, and test against the ordinals and values we are
// expecting, starting with the last expected entity and decrementing
int index = 14;
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//ensure the 'madeup' property is set to true
- assertTrue(Boolean.parseBoolean(activity.get("madeup").toString()));
+ assertTrue(Boolean.parseBoolean(entity.get("madeup").toString()));
//make sure the correct ordinal properties are returned
- assertEquals(index--, Long.parseLong(activity.get("ordinal").toString()));
+ assertEquals(index--, Long.parseLong(entity.get("ordinal").toString()));
}
}
@@ -138,7 +138,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryOrInclusive() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "elderberries";
generateTestEntities(numOfEntities, collectionName);
@@ -146,26 +146,26 @@ public class AndOrQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery("select * where madeup = false OR ordinal >= " + (numOfEntities - numOfEntities / 4))
.setLimit((numOfEntities));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
int index = numOfEntities - 1;
int count = 0;
- int returnSize = activities.getResponse().getEntityCount();
+ int returnSize = coll.getResponse().getEntityCount();
//loop through the returned results
for (int i = 0; i < returnSize; i++, index--) {
count++;
- Entity activity = activities.getResponse().getEntities().get(i);
- logger.info(String.valueOf(activity.get("ordinal")) + " " + String.valueOf(activity.get("madeup")));
+ Entity entity = coll.getResponse().getEntities().get(i);
+ logger.info(String.valueOf(entity.get("ordinal")) + " " + String.valueOf(entity.get("madeup")));
//if the entity is in the first half, the property "madeup" should be false
if (index < numOfEntities / 2) {
- assertFalse(Boolean.parseBoolean(String.valueOf(activity.get("madeup"))));
+ assertFalse(Boolean.parseBoolean(String.valueOf(entity.get("madeup"))));
}
//else if the entity is in the second half, the property "madeup" should be true
else if (index >= (numOfEntities - numOfEntities / 4)) {
- assertTrue(Boolean.parseBoolean(String.valueOf(activity.get("madeup"))));
+ assertTrue(Boolean.parseBoolean(String.valueOf(entity.get("madeup"))));
}
//test to ensure that the ordinal is in the first half (where "madeup = false")
//OR that the ordinal is in the last quarter of the entity list (where "ordinal >= (numOfEntities - numOfEntities / 4))")
- long ordinal = Long.parseLong(String.valueOf(activity.get("ordinal")));
+ long ordinal = Long.parseLong(String.valueOf(entity.get("ordinal")));
assertTrue(ordinal < (numOfEntities / 2) || ordinal >= (numOfEntities - numOfEntities / 4));
}
//results should have madeup = false or ordinal 0-9,15-19
@@ -181,7 +181,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryOrExclusive() throws IOException {
int numOfEntities = 30;
- String collectionName = "activities";
+ String collectionName = "figs";
generateTestEntities(numOfEntities, collectionName);
@@ -189,23 +189,23 @@ public class AndOrQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery("select * where (verb = 'go' OR ordinal >= " + (numOfEntities - numOfEntities / 4) + ") AND NOT (verb = 'go' AND ordinal >= " + (numOfEntities - numOfEntities / 4) + ")")
.setLimit((numOfEntities));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
int index = numOfEntities - 1;
int count = 0;
- int returnSize = activities.getResponse().getEntityCount();
+ int returnSize = coll.getResponse().getEntityCount();
for (int i = 0; i < returnSize; i++, index--) {
count++;
- Entity activity = activities.getResponse().getEntities().get(i);
- long ordinal = Long.parseLong(String.valueOf(activity.get("ordinal")));
- logger.info(ordinal + " " + String.valueOf(activity.get("verb")));
+ Entity entity = coll.getResponse().getEntities().get(i);
+ long ordinal = Long.parseLong(String.valueOf(entity.get("ordinal")));
+ logger.info(ordinal + " " + String.valueOf(entity.get("verb")));
//if the entity is in the first three quarters, the property "verb" should be "go"
if (ordinal < (numOfEntities - numOfEntities / 4)) {
- assertEquals("go", String.valueOf(activity.get("verb")));
+ assertEquals("go", String.valueOf(entity.get("verb")));
}
//if the entity is in the last quarter, the property "verb" should be "stop"
else if (ordinal >= (numOfEntities - numOfEntities / 4)) {
- assertEquals("stop", String.valueOf(activity.get("verb")));
+ assertEquals("stop", String.valueOf(entity.get("verb")));
}
}
//results should be even ordinals in the first 3 quarters and odd ordinals from the last quarter
@@ -225,17 +225,18 @@ public class AndOrQueryTest extends QueryTestBase {
public void queryWithAndPastLimit() throws IOException {
int numValuesTested = 40;
- generateTestEntities(numValuesTested, "activities");
+ String collectionName = "grapes";
+ generateTestEntities(numValuesTested, collectionName);
//3. Query all entities where "madeup = true"
String errorQuery = "select * where madeup = true";
QueryParameters params = new QueryParameters()
.setQuery(errorQuery)
.setLimit(numValuesTested / 2);//4. Limit the query to half of the number of entities
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//5. Ensure the correct entities are returned
- assertEquals(numValuesTested / 2, activities.getResponse().getEntityCount());
- while (activities.hasNext()) {
- assertTrue(Boolean.parseBoolean(activities.next().get("madeup").toString()));
+ assertEquals(numValuesTested / 2, coll.getResponse().getEntityCount());
+ while (coll.hasNext()) {
+ assertTrue(Boolean.parseBoolean(coll.next().get("madeup").toString()));
}
}
@@ -252,16 +253,17 @@ public class AndOrQueryTest extends QueryTestBase {
public void queryNegated() throws IOException {
int numValuesTested = 20;
- generateTestEntities(numValuesTested, "activities");
+ String collectionName = "huckleberries";
+ generateTestEntities(numValuesTested, collectionName);
//1. Query all entities where "NOT verb = 'go'"
String query = "select * where not verb = 'go'";
//2. Limit the query to half of the number of entities
QueryParameters params = new QueryParameters().setQuery(query).setLimit(numValuesTested / 2);
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. Ensure the returned entities have "verb = 'stop'"
- assertEquals(numValuesTested / 2, activities.getResponse().getEntityCount());
- while (activities.hasNext()) {
- assertEquals("stop", activities.next().get("verb").toString());
+ assertEquals(numValuesTested / 2, coll.getResponse().getEntityCount());
+ while (coll.hasNext()) {
+ assertEquals("stop", coll.next().get("verb").toString());
}
@@ -278,15 +280,16 @@ public class AndOrQueryTest extends QueryTestBase {
public void queryReturnCount() throws Exception {
int numValuesTested = 20;
- generateTestEntities(numValuesTested, "activities");
+ String collectionName = "lemons";
+ generateTestEntities(numValuesTested, collectionName);
//1. Query for a subset of the entities
String inCorrectQuery = "select * where ordinal >= " + (numValuesTested / 2) + " order by ordinal asc";
QueryParameters params = new QueryParameters().setQuery(inCorrectQuery).setLimit(numValuesTested / 2);
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//2. Validate that the correct entities are returned
- assertEquals(numValuesTested / 2, activities.getResponse().getEntityCount());
+ assertEquals(numValuesTested / 2, coll.getResponse().getEntityCount());
- List<Entity> entitiesReturned = activities.getResponse().getEntities();
+ List<Entity> entitiesReturned = coll.getResponse().getEntities();
for (int i = 0; i < numValuesTested / 2; i++) {
assertEquals(numValuesTested / 2 + i, Integer.parseInt(entitiesReturned.get(i).get("ordinal").toString()));
}
@@ -303,7 +306,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryCheckAsc() throws Exception {
int numOfEntities = 20;
- String collectionName = "imagination";
+ String collectionName = "melons";
generateTestEntities(numOfEntities, collectionName);
@@ -332,18 +335,18 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryReturnCheck() throws Exception {
int numOfEntities = 20;
- String collectionName = "imagination";
+ String collectionName = "nectarines";
generateTestEntities(numOfEntities, collectionName);
//2. Issue a query
String inquisitiveQuery = String.format("select * where ordinal >= 0 and ordinal <= %d or WhoHelpedYou = 'Ruff' ORDER BY created", numOfEntities);
QueryParameters params = new QueryParameters().setQuery(inquisitiveQuery);
- Collection activities = this.app().collection(collectionName).get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. validate that a full page of (10) entities is returned
- assertEquals(10, activities.getResponse().getEntityCount());
- List<Entity> entitiesReturned = activities.getResponse().getEntities();
+ assertEquals(10, coll.getResponse().getEntityCount());
+ List<Entity> entitiesReturned = coll.getResponse().getEntities();
for (int i = 0; i < 10; i++) {
assertEquals(i, Integer.parseInt(entitiesReturned.get(i).get("ordinal").toString()));
}
@@ -359,7 +362,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryReturnCheckWithShortHand() throws Exception {
int numOfEntities = 10;
- String collectionName = "imagination";
+ String collectionName = "oranges";
generateTestEntities(numOfEntities, collectionName);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d06dbc95/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
index 0692667..950d59d 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
@@ -69,7 +69,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnDoubleQuotes() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "otherthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -95,7 +95,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingQuotes() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "stillotherthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -121,7 +121,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingProperty() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "yetotherthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -147,7 +147,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingPropertyValue() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "thesethings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -173,7 +173,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingOperator() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "thosethings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -200,7 +200,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void limitInQuery() throws IOException {
int numOfEntities =1;
- String collectionName = "things";
+ String collectionName = "whatthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d06dbc95/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
index 6591713..e9bb021 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
@@ -41,21 +41,21 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByLongAsc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "beans";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by ordinal asc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
//results should be ordered by ordinal
int index = 0;
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//make sure the correct ordinal properties are returned
- assertEquals(index++, Long.parseLong(activity.get("ordinal").toString()));
+ assertEquals(index++, Long.parseLong(entity.get("ordinal").toString()));
}
}
@@ -68,23 +68,23 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByLongDesc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "chickpeas";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by ordinal desc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
//Since the sort order is descending, start at the last entity we created
int index = numOfEntities - 1;
//results should be sorted by ordinal
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//make sure the correct ordinal properties are returned
//decrement the index to get the next entity in reverse order
- assertEquals(index--, Long.parseLong(activity.get("ordinal").toString()));
+ assertEquals(index--, Long.parseLong(entity.get("ordinal").toString()));
}
}
@@ -96,25 +96,25 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByBooleanAsc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "lentils";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by madeup asc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted false, then true
//The first half of entities returned should have "madeup = false"
//The second half of entities returned should have "madeup = true"
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
if (index++ < numOfEntities / 2) {
- assertEquals("false", activity.get("madeup").toString());
+ assertEquals("false", entity.get("madeup").toString());
} else {
- assertEquals("true", activity.get("madeup").toString());
+ assertEquals("true", entity.get("madeup").toString());
}
}
}
@@ -127,26 +127,26 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByBooleanDesc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "peas";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by madeup desc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted true, then false
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//make sure the booleans are ordered correctly
//The first half of entities returned should have "madeup = true"
//The second half of entities returned should have "madeup = false"
if (index++ < numOfEntities / 2) {
- assertEquals("true", activity.get("madeup").toString());
+ assertEquals("true", entity.get("madeup").toString());
} else {
- assertEquals("false", activity.get("madeup").toString());
+ assertEquals("false", entity.get("madeup").toString());
}
}
}
@@ -159,25 +159,25 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByStringAsc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "carrots";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
//Sort by the "verb" property to test alphabetical sorting of string properties
QueryParameters params = new QueryParameters()
.setQuery("select * order by verb asc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted "go", then "stop"
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//The first half of entities returned should have "verb = 'go'"
//The second half of entities returned should have "verb = 'stop'"
if (index++ < numOfEntities / 2) {
- assertEquals("go", activity.get("verb").toString());
+ assertEquals("go", entity.get("verb").toString());
} else {
- assertEquals("stop", activity.get("verb").toString());
+ assertEquals("stop", entity.get("verb").toString());
}
}
}
@@ -190,7 +190,7 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByStringDesc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "celery";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -198,18 +198,18 @@ public class OrderByTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery("select * order by verb desc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted "stop", then "go"
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//The first half of entities returned should have "verb = 'stop'"
//The second half of entities returned should have "verb = 'go'"
if (index++ < numOfEntities / 2) {
- assertEquals("stop", activity.get("verb").toString());
+ assertEquals("stop", entity.get("verb").toString());
} else {
- assertEquals("go", activity.get("verb").toString());
+ assertEquals("go", entity.get("verb").toString());
}
}
@@ -229,6 +229,7 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByShouldNotAffectResults() throws IOException {
+ String collectionName = "mushrooms";
long created = 0;
Entity actor = new Entity();
actor.put("displayName", "Erin");
@@ -239,10 +240,10 @@ public class OrderByTest extends QueryTestBase {
//1. Insert entities
for (int i = 0; i < 20; i++) {
props.put("ordinal", i);
- Entity activity = this.app().collection("activity").post(props);
- logger.info("Created", activity.get("created").toString());
+ Entity entity = this.app().collection(collectionName).post(props);
+ logger.info("Created", entity.get("created").toString());
if (i == 5) {
- created = Long.parseLong(activity.get("created").toString());
+ created = Long.parseLong(entity.get("created").toString());
}
}
@@ -250,18 +251,18 @@ public class OrderByTest extends QueryTestBase {
//2. Query without 'order by'
String query = "select * where created > " + created;
QueryParameters params = new QueryParameters().setQuery(query);
- Collection activitiesWithoutOrderBy = this.app().collection("activities").get(params);
- assertEquals(10, activitiesWithoutOrderBy.getResponse().getEntityCount());
+ Collection entitiesWithoutOrderBy = this.app().collection(collectionName).get(params);
+ assertEquals(10, entitiesWithoutOrderBy.getResponse().getEntityCount());
//3. Query with 'order by'
query = query + " order by created desc";
params.setQuery(query);
- Collection activitiesWithOrderBy = this.app().collection("activities").get(params);
+ Collection activitiesWithOrderBy = this.app().collection(collectionName).get(params);
assertEquals(10, activitiesWithOrderBy.getResponse().getEntityCount());
//4. Ensure the same entities are returned
- while (activitiesWithoutOrderBy.hasNext() && activitiesWithOrderBy.hasNext()) {
- Entity activityWithoutOrderBy = activitiesWithoutOrderBy.next();
- Entity activityWithOrderBy = activitiesWithOrderBy.next();
- assertEquals(activityWithoutOrderBy.get("uuid").toString(), activityWithOrderBy.get("uuid").toString());
+ while (entitiesWithoutOrderBy.hasNext() && activitiesWithOrderBy.hasNext()) {
+ Entity entityWithoutOrderBy = entitiesWithoutOrderBy.next();
+ Entity entityWithOrderBy = activitiesWithOrderBy.next();
+ assertEquals(entityWithoutOrderBy.get("uuid").toString(), entityWithOrderBy.get("uuid").toString());
}
}
@@ -277,6 +278,7 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByComesBeforeLimitResult() throws IOException {
+ String collectionName = "onions";
Entity actor = new Entity();
actor.put("displayName", "Erin");
Entity props = new Entity();
@@ -286,23 +288,23 @@ public class OrderByTest extends QueryTestBase {
//1. Insert entities
for (int i = 0; i < 20; i++) {
props.put("ordinal", i);
- this.app().collection("activity").post(props);
+ this.app().collection(collectionName).post(props);
}
waitForQueueDrainAndRefreshIndex();
//2. Query a subset of the entities, specifying order and limit
String query = "select * where created > " + 1 + " order by created desc";
QueryParameters params = new QueryParameters().setQuery(query).setLimit(5);
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. Ensure the correct number of results are returned
- assertEquals(5, activities.getResponse().getEntityCount());
+ assertEquals(5, coll.getResponse().getEntityCount());
//2. Query a subset of the entities, specifying order and limit
query = " where created > " + 1 + " order by created desc";
params = new QueryParameters().setQuery(query).setLimit(5);
- activities = this.app().collection("activities").get(params);
+ coll = this.app().collection(collectionName).get(params);
//3. Ensure the correct number of results are returned
- assertEquals(5, activities.getResponse().getEntityCount());
+ assertEquals(5, coll.getResponse().getEntityCount());
}
/**
@@ -316,8 +318,9 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByReturnCorrectResults() throws IOException {
+ String collectionName = "peppers";
int size = 20;
- Entity[] activities = new Entity[size];
+ Entity[] entities = new Entity[size];
Entity actor = new Entity();
actor.put("displayName", "Erin");
@@ -328,34 +331,34 @@ public class OrderByTest extends QueryTestBase {
//1. Insert a number of entities and add them to an array
for (int i = 0; i < size; i++) {
props.put("ordinal", i);
- Entity e = this.app().collection("activity").post(props);
- activities[i] = e;
+ Entity e = this.app().collection(collectionName).post(props);
+ entities[i] = e;
logger.info(String.valueOf(e.get("uuid").toString()));
- logger.info(String.valueOf(Long.parseLong(activities[0].get("created").toString())));
+ logger.info(String.valueOf(Long.parseLong(entities[0].get("created").toString())));
}
waitForQueueDrainAndRefreshIndex(750);
- ArrayUtils.reverse(activities);
- long lastCreated = Long.parseLong(activities[0].get("created").toString());
+ ArrayUtils.reverse(entities);
+ long lastCreated = Long.parseLong(entities[0].get("created").toString());
//2. Query for the entities in descending order
String errorQuery = String.format("select * where created <= %d order by created desc", lastCreated);
int index = 0;
QueryParameters params = new QueryParameters().setQuery(errorQuery);
- Collection activitiesResponse = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. Validate that the order is correct
do {
- int returnSize = activitiesResponse.getResponse().getEntityCount();
+ int returnSize = coll.getResponse().getEntityCount();
//loop through the current page of results
for (int i = 0; i < returnSize; i++, index++) {
- assertEquals( ( activities[index] ).get( "uuid" ).toString(),
- activitiesResponse.getResponse().getEntities().get(i).get("uuid").toString());
+ assertEquals( ( entities[index] ).get( "uuid" ).toString(),
+ coll.getResponse().getEntities().get(i).get("uuid").toString());
}
//grab the next page of results
- activitiesResponse = this.app().collection("activities").getNextPage(activitiesResponse, params, true);
+ coll = this.app().collection(collectionName).getNextPage(coll, params, true);
}
- while (activitiesResponse.getCursor() != null);
+ while (coll.getCursor() != null);
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d06dbc95/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
index acf51c1..cb103cf 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
@@ -41,14 +41,15 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testNestedSelectFieldNames() throws Exception {
- generateTestEntities(20, "things");
+ String collectionName = "basketballs";
+ generateTestEntities(20, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select actor.displayName,sometestprop where sometestprop = 'testprop'");
- Collection things = this.app().collection("things").get(params);
- assertEquals( 10, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals( 10, coll.getNumOfEntities() );
- Iterator<Entity> iter = things.iterator();
+ Iterator<Entity> iter = coll.iterator();
while ( iter.hasNext() ) {
Entity entity = iter.next();
@@ -73,7 +74,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testMixedCaseDupField() throws Exception {
- String collectionName = "things";
+ String collectionName = "baseballs";
String value = RandomStringUtils.randomAlphabetic( 20 );
String otherValue = RandomStringUtils.randomAlphabetic( 20 );
@@ -89,19 +90,19 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + otherValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
params = new QueryParameters()
.setQuery( "select * where TESTPROP='" + otherValue + "'" );
- things = app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
@Test
public void testStringWithSingleQuote() throws Exception {
- String collectionName = "things";
+ String collectionName = "footballs";
String value = "test'value";
String escapedValue = "test\\'value";
@@ -116,14 +117,14 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testprop='" + escapedValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
+ Collection things = this.app().collection(collectionName).get( params );
assertEquals( 1, things.getNumOfEntities() );
}
@Test
public void testStringWithPlus() throws Exception {
- String collectionName = "things";
+ String collectionName = "volleyballs";
String value = "ed+test@usergrid.com";
// create entity with value containing a plus symbol
@@ -135,15 +136,15 @@ public class SelectMappingsQueryTest extends QueryTestBase {
// now query this without encoding the plus symbol
QueryParameters params = new QueryParameters()
.setQuery( "select * where testprop='" + value + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
// again query with the plus symbol url encoded
String escapedValue = "ed%2Btest@usergrid.com";
params = new QueryParameters()
.setQuery( "select * where testprop='" + escapedValue + "'" );
- things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
@@ -154,7 +155,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testFieldOverride1() throws Exception {
- String collectionName = "things";
+ String collectionName = "pickleballs";
// create entity with testProp=value
String value = RandomStringUtils.randomAlphabetic( 20 );
@@ -172,13 +173,13 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + newValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
params = new QueryParameters()
.setQuery( "select * where TESTPROP='" + newValue + "'" );
- things = app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
/**
@@ -187,7 +188,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testFieldOverride2() throws Exception {
- String collectionName = "things";
+ String collectionName = "tennisballs";
// create entity with TESTPROP=value
String value = RandomStringUtils.randomAlphabetic( 20 );
@@ -205,13 +206,13 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + newValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
params = new QueryParameters()
.setQuery( "select * where TESTPROP='" + newValue + "'" );
- things = app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
}
[10/15] usergrid git commit: move deletes to new delete queue -- read
repair will fix attempts to access deleted entities and connections,
so indexing and collection deletes can proceed more slowly than other types of
changes
Posted by md...@apache.org.
move deletes to new delete queue -- read repair will fix attempts to access deleted entities and connections, so indexing and collection deletes can proceed more slowly than other types of changes
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/570e1ab4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/570e1ab4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/570e1ab4
Branch: refs/heads/master
Commit: 570e1ab4e2d4d356756fedee6b7ed5a1bd2d8e93
Parents: 39ec4f2
Author: Mike Dunker <md...@google.com>
Authored: Fri Aug 18 15:12:56 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Fri Aug 18 15:12:56 2017 -0700
----------------------------------------------------------------------
.../asyncevents/AsyncEventQueueType.java | 35 ++
.../asyncevents/AsyncEventService.java | 4 +-
.../asyncevents/AsyncEventServiceImpl.java | 463 +++++++++----------
.../index/IndexProcessorFig.java | 18 +
.../corepersistence/index/ReIndexAction.java | 5 +-
.../index/ReIndexServiceImpl.java | 3 +-
.../read/traverse/AbstractReadGraphFilter.java | 11 +-
.../AbstractReadReverseGraphFilter.java | 11 +-
8 files changed, 292 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
new file mode 100644
index 0000000..4b91e17
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.asyncevents;
+
+public enum AsyncEventQueueType {
+ REGULAR ("regular"), UTILITY("utility"), DELETE("delete");
+
+ private String displayName;
+ AsyncEventQueueType(String displayName) {
+ this.displayName = displayName;
+ }
+
+ @Override
+ public String toString() {
+ return displayName;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index cab4e3e..9e346cf 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -84,9 +84,9 @@ public interface AsyncEventService extends ReIndexAction {
/**
*
* @param indexOperationMessage
- * @param forUtilityQueue
+ * @param queueType
*/
- void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage, boolean forUtilityQueue);
+ void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage, AsyncEventQueueType queueType);
/**
* @param applicationScope
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 9501ad3..0e55e9b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -59,6 +59,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.persistence.queue.*;
import org.apache.usergrid.persistence.queue.impl.LegacyQueueScopeImpl;
+import org.apache.usergrid.persistence.queue.impl.SNSQueueManagerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
@@ -104,13 +105,16 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public int MAX_TAKE = 10;
public static final String QUEUE_NAME = "index"; //keep this short as AWS limits queue name size to 80 chars
public static final String QUEUE_NAME_UTILITY = "utility"; //keep this short as AWS limits queue name size to 80 chars
+ public static final String QUEUE_NAME_DELETE = "delete";
public static final String DEAD_LETTER_SUFFIX = "_dead";
private final LegacyQueueManager indexQueue;
private final LegacyQueueManager utilityQueue;
+ private final LegacyQueueManager deleteQueue;
private final LegacyQueueManager indexQueueDead;
private final LegacyQueueManager utilityQueueDead;
+ private final LegacyQueueManager deleteQueueDead;
private final IndexProcessorFig indexProcessorFig;
private final LegacyQueueFig queueFig;
private final IndexProducer indexProducer;
@@ -132,8 +136,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
private final Counter indexErrorCounter;
private final AtomicLong counter = new AtomicLong();
private final AtomicLong counterUtility = new AtomicLong();
+ private final AtomicLong counterDelete = new AtomicLong();
private final AtomicLong counterIndexDead = new AtomicLong();
private final AtomicLong counterUtilityDead = new AtomicLong();
+ private final AtomicLong counterDeleteDead = new AtomicLong();
private final AtomicLong inFlight = new AtomicLong();
private final Histogram messageCycle;
private final MapManager esMapPersistence;
@@ -174,16 +180,24 @@ public class AsyncEventServiceImpl implements AsyncEventService {
LegacyQueueScope utilityQueueScope =
new LegacyQueueScopeImpl(QUEUE_NAME_UTILITY, LegacyQueueScope.RegionImplementation.ALL);
+ LegacyQueueScope deleteQueueScope =
+ new LegacyQueueScopeImpl(QUEUE_NAME_DELETE, LegacyQueueScope.RegionImplementation.ALL);
+
LegacyQueueScope indexQueueDeadScope =
new LegacyQueueScopeImpl(QUEUE_NAME, LegacyQueueScope.RegionImplementation.ALL, true);
LegacyQueueScope utilityQueueDeadScope =
new LegacyQueueScopeImpl(QUEUE_NAME_UTILITY, LegacyQueueScope.RegionImplementation.ALL, true);
+ LegacyQueueScope deleteQueueDeadScope =
+ new LegacyQueueScopeImpl(QUEUE_NAME_DELETE, LegacyQueueScope.RegionImplementation.ALL, true);
+
this.indexQueue = queueManagerFactory.getQueueManager(indexQueueScope);
this.utilityQueue = queueManagerFactory.getQueueManager(utilityQueueScope);
+ this.deleteQueue = queueManagerFactory.getQueueManager(deleteQueueScope);
this.indexQueueDead = queueManagerFactory.getQueueManager(indexQueueDeadScope);
this.utilityQueueDead = queueManagerFactory.getQueueManager(utilityQueueDeadScope);
+ this.deleteQueueDead = queueManagerFactory.getQueueManager(deleteQueueDeadScope);
this.indexProcessorFig = indexProcessorFig;
this.queueFig = queueFig;
@@ -206,34 +220,90 @@ public class AsyncEventServiceImpl implements AsyncEventService {
start();
}
+ private String getQueueName(AsyncEventQueueType queueType) {
+ switch (queueType) {
+ case REGULAR:
+ return QUEUE_NAME;
+
+ case UTILITY:
+ return QUEUE_NAME_UTILITY;
+
+ case DELETE:
+ return QUEUE_NAME_DELETE;
+
+ default:
+ throw new IllegalArgumentException("Invalid queue type: " + queueType.toString());
+ }
+ }
+
+ private LegacyQueueManager getQueue(AsyncEventQueueType queueType) {
+ return getQueue(queueType, false);
+ }
+
+ private LegacyQueueManager getQueue(AsyncEventQueueType queueType, boolean isDeadQueue) {
+ switch (queueType) {
+ case REGULAR:
+ return isDeadQueue ? indexQueueDead : indexQueue;
+
+ case UTILITY:
+ return isDeadQueue ? utilityQueueDead : utilityQueue;
+
+ case DELETE:
+ return isDeadQueue ? deleteQueueDead : deleteQueue;
+
+ default:
+ throw new IllegalArgumentException("Invalid queue type: " + queueType.toString());
+ }
+ }
+
+ private AtomicLong getCounter(AsyncEventQueueType queueType, boolean isDeadQueue) {
+ switch (queueType) {
+ case REGULAR:
+ return isDeadQueue ? counterIndexDead : counter;
+
+ case UTILITY:
+ return isDeadQueue ? counterUtilityDead : counterUtility;
+
+ case DELETE:
+ return isDeadQueue ? counterDeleteDead : counterDelete;
+
+ default:
+ throw new IllegalArgumentException("Invalid queue type: " + queueType.toString());
+ }
+ }
+
+
+
+
/**
* Offer the EntityIdScope to SQS
*/
private void offer(final Serializable operation) {
+ offer(operation, AsyncEventQueueType.REGULAR);
+ }
+
+ private void offer(final Serializable operation, AsyncEventQueueType queueType) {
final Timer.Context timer = this.writeTimer.time();
try {
//signal to SQS
- this.indexQueue.sendMessageToLocalRegion( operation );
+ getQueue(queueType).sendMessageToLocalRegion(operation);
} catch (IOException e) {
throw new RuntimeException("Unable to queue message", e);
} finally {
timer.stop();
}
+
}
- private void offerTopic(final Serializable operation, boolean forUtilityQueue) {
+ private void offerTopic(final Serializable operation, AsyncEventQueueType queueType) {
final Timer.Context timer = this.writeTimer.time();
try {
//signal to SQS
- if (forUtilityQueue) {
- this.utilityQueue.sendMessageToAllRegions(operation);
- } else {
- this.indexQueue.sendMessageToAllRegions(operation);
- }
+ getQueue(queueType).sendMessageToAllRegions(operation);
}
catch ( IOException e ) {
throw new RuntimeException( "Unable to queue message", e );
@@ -244,15 +314,11 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
- private void offerBatch(final List operations, boolean forUtilityQueue){
+ private void offerBatch(final List operations, AsyncEventQueueType queueType){
final Timer.Context timer = this.writeTimer.time();
try {
//signal to SQS
- if( forUtilityQueue ){
- this.utilityQueue.sendMessages(operations);
- }else{
- this.indexQueue.sendMessages(operations);
- }
+ getQueue(queueType).sendMessages(operations);
} catch (IOException e) {
throw new RuntimeException("Unable to queue message", e);
} finally {
@@ -260,25 +326,16 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
}
- private void offerBatchToUtilityQueue(final List operations){
- try {
- //signal to SQS
- this.utilityQueue.sendMessages(operations);
- } catch (IOException e) {
- throw new RuntimeException("Unable to queue message", e);
- }
- }
-
/**
* Take message
*/
- private List<LegacyQueueMessage> take() {
+ private List<LegacyQueueMessage> take(AsyncEventQueueType queueType, boolean isDeadQueue) {
final Timer.Context timer = this.readTimer.time();
try {
- return indexQueue.getMessages(MAX_TAKE, AsyncEvent.class);
+ return getQueue(queueType, isDeadQueue).getMessages(MAX_TAKE, AsyncEvent.class);
}
finally {
//stop our timer
@@ -286,52 +343,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
}
- /**
- * Take message from SQS utility queue
- */
- private List<LegacyQueueMessage> takeFromUtilityQueue() {
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
- return utilityQueue.getMessages(MAX_TAKE, AsyncEvent.class);
- }
- finally {
- //stop our timer
- timer.stop();
- }
- }
-
- /**
- * Take message from index dead letter queue
- */
- private List<LegacyQueueMessage> takeFromIndexDeadQueue() {
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
- return indexQueueDead.getMessages(MAX_TAKE, AsyncEvent.class);
- }
- finally {
- //stop our timer
- timer.stop();
- }
- }
-
- /**
- * Take message from SQS utility dead letter queue
- */
- private List<LegacyQueueMessage> takeFromUtilityDeadQueue() {
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
- return utilityQueueDead.getMessages(MAX_TAKE, AsyncEvent.class);
- }
- finally {
- //stop our timer
- timer.stop();
- }
+ private List<LegacyQueueMessage> take(AsyncEventQueueType queueType) {
+ return take(queueType, false);
}
@@ -362,38 +375,15 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
}
-
- /**
- * calls the event handlers and returns a result with information on whether
- * it needs to be ack'd and whether it needs to be indexed
- * Ack message in SQS
- */
- public void ackUtilityQueue(final List<LegacyQueueMessage> messages) {
- try{
- utilityQueue.commitMessages( messages );
- }catch(Exception e){
- throw new RuntimeException("Unable to ack messages", e);
+ public void ack(final List<LegacyQueueMessage> messages, AsyncEventQueueType queueType, boolean isDeadQueue) {
+ if (queueType == AsyncEventQueueType.REGULAR && !isDeadQueue) {
+ // different functionality
+ ack(messages);
}
- }
-
- /**
- * ack messages in index dead letter queue
- */
- public void ackIndexDeadQueue(final List<LegacyQueueMessage> messages) {
- try{
- indexQueueDead.commitMessages( messages );
- }catch(Exception e){
- throw new RuntimeException("Unable to ack messages", e);
+ try {
+ getQueue(queueType, isDeadQueue).commitMessages( messages );
}
- }
-
- /**
- * ack messages in utility dead letter queue
- */
- public void ackUtilityDeadQueue(final List<LegacyQueueMessage> messages) {
- try{
- utilityQueueDead.commitMessages( messages );
- }catch(Exception e){
+ catch (Exception e) {
throw new RuntimeException("Unable to ack messages", e);
}
}
@@ -536,7 +526,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
applicationScope.getApplication().getUuid(), applicationScope.getApplication().getType());
offerTopic( new InitializeApplicationIndexEvent( queueFig.getPrimaryRegion(),
- new ReplicatedIndexLocationStrategy( indexLocationStrategy ) ), false);
+ new ReplicatedIndexLocationStrategy( indexLocationStrategy ) ), AsyncEventQueueType.REGULAR);
}
@@ -626,7 +616,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
edge.getType(), edge.getTargetNode().getUuid(), edge.getTargetNode().getType());
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
- offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ) );
+ offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ), AsyncEventQueueType.DELETE );
}
private IndexOperationMessage handleEdgeDelete(final LegacyQueueMessage message) {
@@ -660,9 +650,9 @@ public class AsyncEventServiceImpl implements AsyncEventService {
/**
* Queue up an indexOperationMessage for multi region execution
* @param indexOperationMessage
- * @param forUtilityQueue
+ * @param queueType
*/
- public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage, boolean forUtilityQueue) {
+ public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage, AsyncEventQueueType queueType) {
// don't try to produce something with nothing
if(indexOperationMessage == null || indexOperationMessage.isEmpty()){
@@ -688,7 +678,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
logger.trace("Offering ElasticsearchIndexEvent for message {}", newMessageId );
- offerTopic( elasticsearchIndexEvent, forUtilityQueue );
+ offerTopic( elasticsearchIndexEvent, queueType );
}
private void handleIndexOperation(final ElasticsearchIndexEvent elasticsearchIndexEvent)
@@ -764,7 +754,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
applicationScope.getApplication().getUuid(), entityId.getUuid(), entityId.getType());
offerTopic( new DeIndexOldVersionsEvent( queueFig.getPrimaryRegion(),
- new EntityIdScope( applicationScope, entityId), markedVersion), false);
+ new EntityIdScope( applicationScope, entityId), markedVersion), AsyncEventQueueType.DELETE );
}
@@ -824,7 +814,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
logger.trace("Offering EntityDeleteEvent for {}:{}", entityId.getUuid(), entityId.getType());
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
- offer( new EntityDeleteEvent(queueFig.getPrimaryRegion(), new EntityIdScope( applicationScope, entityId ) ) );
+ offer( new EntityDeleteEvent(queueFig.getPrimaryRegion(), new EntityIdScope( applicationScope, entityId ) ),
+ AsyncEventQueueType.DELETE );
}
private IndexOperationMessage handleEntityDelete(final LegacyQueueMessage message) {
@@ -879,25 +870,39 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public void start() {
final int indexCount = indexProcessorFig.getWorkerCount();
final int utilityCount = indexProcessorFig.getWorkerCountUtility();
+ final int deleteCount = indexProcessorFig.getWorkerCountDelete();
final int indexDeadCount = indexProcessorFig.getWorkerCountDeadLetter();
final int utilityDeadCount = indexProcessorFig.getWorkerCountUtilityDeadLetter();
- logger.info("Starting queue workers for indexing: index={} indexDLQ={} utility={} utilityDLQ={}", indexCount, indexDeadCount,
- utilityCount, utilityDeadCount);
+ final int deleteDeadCount = indexProcessorFig.getWorkerCountDeleteDeadLetter();
+ logger.info("Starting queue workers for indexing: index={} indexDLQ={} utility={} utilityDLQ={} delete={} deleteDLQ={}",
+ indexCount, indexDeadCount, utilityCount, utilityDeadCount, deleteCount, deleteDeadCount);
for (int i = 0; i < indexCount; i++) {
- startWorker(QUEUE_NAME);
+ startWorker(AsyncEventQueueType.REGULAR);
}
for (int i = 0; i < utilityCount; i++) {
- startWorker(QUEUE_NAME_UTILITY);
+ startWorker(AsyncEventQueueType.UTILITY);
}
- for (int i = 0; i < indexDeadCount; i++) {
- startDeadQueueWorker(QUEUE_NAME);
+ for (int i = 0; i < deleteCount; i++) {
+ startWorker(AsyncEventQueueType.DELETE);
}
+ if( indexQueue instanceof SNSQueueManagerImpl) {
+ logger.info("Queue manager implementation supports dead letters, start dead letter queue workers.");
+ for (int i = 0; i < indexDeadCount; i++) {
+ startDeadQueueWorker(AsyncEventQueueType.REGULAR);
+ }
- for (int i = 0; i < utilityDeadCount; i++) {
- startDeadQueueWorker(QUEUE_NAME_UTILITY);
+ for (int i = 0; i < utilityDeadCount; i++) {
+ startDeadQueueWorker(AsyncEventQueueType.UTILITY);
+ }
+
+ for (int i = 0; i < deleteDeadCount; i++) {
+ startDeadQueueWorker(AsyncEventQueueType.DELETE);
+ }
+ }else{
+ logger.info("Queue manager implementation does NOT support dead letters, NOT starting dead letter queue worker.");
}
}
@@ -916,11 +921,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
- private void startWorker(final String type) {
- Preconditions.checkNotNull(type, "Worker type required");
+ private void startWorker(final AsyncEventQueueType queueType) {
synchronized (mutex) {
- boolean isUtilityQueue = isNotEmpty(type) && type.toLowerCase().contains(QUEUE_NAME_UTILITY.toLowerCase());
+ String type = getQueueName(queueType);
Observable<List<LegacyQueueMessage>> consumer =
Observable.create( new Observable.OnSubscribe<List<LegacyQueueMessage>>() {
@@ -928,20 +932,15 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public void call( final Subscriber<? super List<LegacyQueueMessage>> subscriber ) {
//name our thread so it's easy to see
- long threadNum = isUtilityQueue ?
- counterUtility.incrementAndGet() : counter.incrementAndGet();
- Thread.currentThread().setName( "QueueConsumer_" + type+ "_" + threadNum );
+ long threadNum = getCounter(queueType, false).incrementAndGet();
+ Thread.currentThread().setName( "QueueConsumer_" + type + "_" + threadNum );
List<LegacyQueueMessage> drainList = null;
do {
try {
- if ( isUtilityQueue ){
- drainList = takeFromUtilityQueue();
- }else{
- drainList = take();
+ drainList = take(queueType);
- }
//emit our list in it's entity to hand off to a worker pool
subscriber.onNext(drainList);
@@ -997,7 +996,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// submit the processed messages to index producer
List<LegacyQueueMessage> messagesToAck =
- submitToIndex( indexEventResults, isUtilityQueue );
+ submitToIndex( indexEventResults, queueType );
if ( messagesToAck.size() < messages.size() ) {
logger.warn(
@@ -1007,12 +1006,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// ack each message if making it to this point
if( messagesToAck.size() > 0 ){
-
- if ( isUtilityQueue ){
- ackUtilityQueue( messagesToAck );
- }else{
- ack( messagesToAck );
- }
+ ack(messagesToAck, queueType, false);
}
return messagesToAck;
@@ -1036,129 +1030,112 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
- private void startDeadQueueWorker(final String type) {
- Preconditions.checkNotNull(type, "Worker type required");
+ private void startDeadQueueWorker(final AsyncEventQueueType queueType) {
+ String type = getQueueName(queueType);
synchronized (mutex) {
- boolean isUtilityDeadQueue = isNotEmpty(type) && type.toLowerCase().contains(QUEUE_NAME_UTILITY.toLowerCase());
-
Observable<List<LegacyQueueMessage>> consumer =
- Observable.create( new Observable.OnSubscribe<List<LegacyQueueMessage>>() {
- @Override
- public void call( final Subscriber<? super List<LegacyQueueMessage>> subscriber ) {
+ Observable.create( new Observable.OnSubscribe<List<LegacyQueueMessage>>() {
+ @Override
+ public void call( final Subscriber<? super List<LegacyQueueMessage>> subscriber ) {
- //name our thread so it's easy to see
- long threadNum = isUtilityDeadQueue ?
- counterUtilityDead.incrementAndGet() : counterIndexDead.incrementAndGet();
- Thread.currentThread().setName( "QueueDeadLetterConsumer_" + type+ "_" + threadNum );
+ //name our thread so it's easy to see
+ long threadNum = getCounter(queueType, true).incrementAndGet();
+ Thread.currentThread().setName( "QueueDeadLetterConsumer_" + type + "_" + threadNum );
- List<LegacyQueueMessage> drainList = null;
+ List<LegacyQueueMessage> drainList = null;
- do {
- try {
- if ( isUtilityDeadQueue ){
- drainList = takeFromUtilityDeadQueue();
- }else{
- drainList = takeFromIndexDeadQueue();
- }
- //emit our list in it's entity to hand off to a worker pool
- subscriber.onNext(drainList);
+ do {
+ try {
+ drainList = take(queueType, true);
- //take since we're in flight
- inFlight.addAndGet( drainList.size() );
+ //emit our list in it's entity to hand off to a worker pool
+ subscriber.onNext(drainList);
- } catch ( Throwable t ) {
+ //take since we're in flight
+ inFlight.addAndGet( drainList.size() );
- final long sleepTime = indexProcessorFig.getDeadLetterFailureRetryTime();
+ } catch ( Throwable t ) {
- // there might be an error here during tests, just clean the cache
- indexQueueDead.clearQueueNameCache();
+ final long sleepTime = indexProcessorFig.getDeadLetterFailureRetryTime();
- if ( t instanceof InvalidQueryException ) {
+ // there might be an error here during tests, just clean the cache
+ indexQueueDead.clearQueueNameCache();
- // don't fill up log with exceptions when keyspace and column
- // families are not ready during bootstrap/setup
- logger.warn( "Failed to dequeue dead letters due to '{}'. Sleeping for {} ms",
- t.getMessage(), sleepTime );
+ if ( t instanceof InvalidQueryException ) {
- } else {
- logger.error( "Failed to dequeue dead letters. Sleeping for {} ms", sleepTime, t);
- }
+ // don't fill up log with exceptions when keyspace and column
+ // families are not ready during bootstrap/setup
+ logger.warn( "Failed to dequeue dead letters due to '{}'. Sleeping for {} ms",
+ t.getMessage(), sleepTime );
- if ( drainList != null ) {
- inFlight.addAndGet( -1 * drainList.size() );
- }
+ } else {
+ logger.error( "Failed to dequeue dead letters. Sleeping for {} ms", sleepTime, t);
+ }
- try { Thread.sleep( sleepTime ); } catch ( InterruptedException ie ) {}
+ if ( drainList != null ) {
+ inFlight.addAndGet( -1 * drainList.size() );
}
+
+ try { Thread.sleep( sleepTime ); } catch ( InterruptedException ie ) {}
}
- while ( true );
}
- } ) //this won't block our read loop, just reads and proceeds
- .flatMap( sqsMessages -> {
-
- //do this on a different schedule, and introduce concurrency
- // with flatmap for faster processing
- return Observable.just( sqsMessages )
-
- .map( messages -> {
- if ( messages == null || messages.size() == 0 ) {
- // no messages came from the queue, move on
- return null;
- }
-
- try {
- // put the dead letter messages back in the appropriate queue
- LegacyQueueManager returnQueue = null;
- String queueType;
- if (isUtilityDeadQueue) {
- returnQueue = utilityQueue;
- queueType = "utility";
- } else {
- returnQueue = indexQueue;
- queueType = "index";
- }
- List<LegacyQueueMessage> successMessages = returnQueue.sendQueueMessages(messages);
- for (LegacyQueueMessage msg : successMessages) {
- logger.warn("Returning message to {} queue: type:{}, messageId:{} body: {}", queueType, msg.getType(), msg.getMessageId(), msg.getStringBody());
- }
- int unsuccessfulMessagesSize = messages.size() - successMessages.size();
- if (unsuccessfulMessagesSize > 0) {
- // some messages couldn't be sent to originating queue, log
- Set<String> successMessageIds = new HashSet<>();
- for (LegacyQueueMessage msg : successMessages) {
- String messageId = msg.getMessageId();
- if (successMessageIds.contains(messageId)) {
- logger.warn("Found duplicate messageId in returned messages: {}", messageId);
- } else {
- successMessageIds.add(messageId);
- }
- }
- for (LegacyQueueMessage msg : messages) {
- String messageId = msg.getMessageId();
- if (!successMessageIds.contains(messageId)) {
- logger.warn("Failed to return message to {} queue: type:{} messageId:{} body: {}", queueType, msg.getType(), messageId, msg.getStringBody());
- }
- }
- }
-
- if (isUtilityDeadQueue) {
- ackUtilityDeadQueue(successMessages);
- } else {
- ackIndexDeadQueue(successMessages);
- }
-
- return messages;
- }
- catch ( Exception e ) {
- logger.error( "Failed to ack messages", e );
- return null;
- //do not rethrow so we can process all of them
- }
- } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() );
-
- //end flatMap
- }, indexProcessorFig.getEventConcurrencyFactor() );
+ while ( true );
+ }
+ } ) //this won't block our read loop, just reads and proceeds
+ .flatMap( sqsMessages -> {
+
+ //do this on a different schedule, and introduce concurrency
+ // with flatmap for faster processing
+ return Observable.just( sqsMessages )
+
+ .map( messages -> {
+ if ( messages == null || messages.size() == 0 ) {
+ // no messages came from the queue, move on
+ return null;
+ }
+
+ try {
+ // put the dead letter messages back in the appropriate queue
+ LegacyQueueManager returnQueue = getQueue(queueType, false);
+
+ List<LegacyQueueMessage> successMessages = returnQueue.sendQueueMessages(messages);
+ for (LegacyQueueMessage msg : successMessages) {
+ logger.warn("Returning message to {} queue: type:{}, messageId:{} body: {}", queueType.toString(), msg.getType(), msg.getMessageId(), msg.getStringBody());
+ }
+ int unsuccessfulMessagesSize = messages.size() - successMessages.size();
+ if (unsuccessfulMessagesSize > 0) {
+ // some messages couldn't be sent to originating queue, log
+ Set<String> successMessageIds = new HashSet<>();
+ for (LegacyQueueMessage msg : successMessages) {
+ String messageId = msg.getMessageId();
+ if (successMessageIds.contains(messageId)) {
+ logger.warn("Found duplicate messageId in returned messages: {}", messageId);
+ } else {
+ successMessageIds.add(messageId);
+ }
+ }
+ for (LegacyQueueMessage msg : messages) {
+ String messageId = msg.getMessageId();
+ if (!successMessageIds.contains(messageId)) {
+ logger.warn("Failed to return message to {} queue: type:{} messageId:{} body: {}", queueType.toString(), msg.getType(), messageId, msg.getStringBody());
+ }
+ }
+ }
+
+ ack(successMessages, queueType, true);
+
+ return messages;
+ }
+ catch ( Exception e ) {
+ logger.error( "Failed to ack messages", e );
+ return null;
+ //do not rethrow so we can process all of them
+ }
+ } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() );
+
+ //end flatMap
+ }, indexProcessorFig.getEventConcurrencyFactor() );
//start in the background
@@ -1172,7 +1149,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
* Submit results to index and return the queue messages to be ack'd
*
*/
- private List<LegacyQueueMessage> submitToIndex(List<IndexEventResult> indexEventResults, boolean forUtilityQueue) {
+ private List<LegacyQueueMessage> submitToIndex(List<IndexEventResult> indexEventResults, AsyncEventQueueType queueType) {
// if nothing came back then return empty list
if(indexEventResults==null){
@@ -1199,7 +1176,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// collect into a list of QueueMessages that can be ack'd later
.collect(Collectors.toList());
- queueIndexOperationMessage(combined, forUtilityQueue);
+ queueIndexOperationMessage(combined, queueType);
return queueMessages;
}
@@ -1210,10 +1187,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
new EntityIndexOperation( applicationScope, id, updatedSince);
queueIndexOperationMessage(
- eventBuilder.buildEntityIndex( entityIndexOperation ).toBlocking().lastOrDefault(null), false);
+ eventBuilder.buildEntityIndex( entityIndexOperation ).toBlocking().lastOrDefault(null), AsyncEventQueueType.REGULAR );
}
- public void indexBatch(final List<EdgeScope> edges, final long updatedSince, boolean forUtilityQueue) {
+ public void indexBatch(final List<EdgeScope> edges, final long updatedSince, AsyncEventQueueType queueType) {
final List<EntityIndexEvent> batch = new ArrayList<>();
edges.forEach(e -> {
@@ -1226,7 +1203,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
logger.trace("Offering batch of EntityIndexEvent of size {}", batch.size());
- offerBatch( batch, forUtilityQueue );
+ offerBatch( batch, queueType );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 7eecf04..eb63056 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -40,10 +40,14 @@ public interface IndexProcessorFig extends GuicyFig {
String ELASTICSEARCH_WORKER_COUNT_UTILITY = "elasticsearch.worker_count_utility";
+ String ELASTICSEARCH_WORKER_COUNT_DELETE = "elasticsearch.worker_count_delete";
+
String ELASTICSEARCH_WORKER_COUNT_DEADLETTER = "elasticsearch.worker_count_deadletter";
String ELASTICSEARCH_WORKER_COUNT_UTILITY_DEADLETTER = "elasticsearch.worker_count_utility_deadletter";
+ String ELASTICSEARCH_WORKER_COUNT_DELETE_DEADLETTER = "elasticsearch.worker_count_delete_deadletter";
+
String EVENT_CONCURRENCY_FACTOR = "event.concurrency.factor";
String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
@@ -105,6 +109,13 @@ public interface IndexProcessorFig extends GuicyFig {
int getWorkerCountUtility();
/**
+ * The number of worker threads used to read delete requests from the queue.
+ */
+ @Default("1")
+ @Key(ELASTICSEARCH_WORKER_COUNT_DELETE)
+ int getWorkerCountDelete();
+
+ /**
* The number of worker threads used to read dead messages from the index dead letter queue and reload them into the index queue.
*/
@Default("1")
@@ -119,6 +130,13 @@ public interface IndexProcessorFig extends GuicyFig {
int getWorkerCountUtilityDeadLetter();
/**
+ * The number of worker threads used to read dead messages from the delete dead letter queue and reload them into the delete queue.
+ */
+ @Default("1")
+ @Key(ELASTICSEARCH_WORKER_COUNT_DELETE_DEADLETTER)
+ int getWorkerCountDeleteDeadLetter();
+
+ /**
* Set the implementation to use for queuing.
* Valid values: TEST, LOCAL, SQS, SNS
* NOTE: SQS and SNS equate to the same implementation of Amazon queue services.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
index 2b3573e..d6bdd93 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.index;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -42,7 +43,7 @@ public interface ReIndexAction {
* Index a batch list of entities. Goes to the utility queue.
* @param edges
* @param updatedSince
- * @param forUtilityQueue
+ * @param queueType
*/
- void indexBatch(final List<EdgeScope> edges, final long updatedSince, boolean forUtilityQueue);
+ void indexBatch(final List<EdgeScope> edges, final long updatedSince, AsyncEventQueueType queueType);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 0660d5e..c7371b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -169,7 +170,7 @@ public class ReIndexServiceImpl implements ReIndexService {
.buffer( indexProcessorFig.getReindexBufferSize())
.doOnNext( edgeScopes -> {
logger.info("Sending batch of {} to be indexed.", edgeScopes.size());
- indexService.indexBatch(edgeScopes, modifiedSince, true);
+ indexService.indexBatch(edgeScopes, modifiedSince, AsyncEventQueueType.UTILITY);
count.addAndGet(edgeScopes.size() );
if( edgeScopes.size() > 0 ) {
writeCursorState(jobId, edgeScopes.get(edgeScopes.size() - 1));
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 83f4c8b..4886b08 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
@@ -124,7 +125,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
indexMessageObservable
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -139,7 +140,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
entityDeleteResults.getIndexObservable()
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -159,7 +160,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
entityDeleteResults.getIndexObservable()
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -252,13 +253,13 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
}
}
- private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector() {
+ private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector(AsyncEventQueueType queueType) {
return observable -> observable
.collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
.filter(msg -> !msg.isEmpty())
.doOnNext(indexOperation -> {
- asyncEventService.queueIndexOperationMessage(indexOperation, false);
+ asyncEventService.queueIndexOperationMessage(indexOperation, queueType);
});
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
index 1afb524..e8a20dd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
@@ -124,7 +125,7 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
indexMessageObservable
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -139,7 +140,7 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
entityDeleteResults.getIndexObservable()
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -159,7 +160,7 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
entityDeleteResults.getIndexObservable()
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -245,13 +246,13 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
}
}
- private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector() {
+ private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector(AsyncEventQueueType queueType) {
return observable -> observable
.collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
.filter(msg -> !msg.isEmpty())
.doOnNext(indexOperation -> {
- asyncEventService.queueIndexOperationMessage(indexOperation, false);
+ asyncEventService.queueIndexOperationMessage(indexOperation, queueType);
});
}