You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2016/04/19 18:50:58 UTC
[01/50] usergrid git commit: Ignore non critical tests for now.
Repository: usergrid
Updated Branches:
refs/heads/master e0931be42 -> 9950af768
Ignore non critical tests for now.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a6cfda8a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a6cfda8a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a6cfda8a
Branch: refs/heads/master
Commit: a6cfda8ae0ca9dabcd6d18a966eb9848d91f2749
Parents: d7119a4
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Apr 4 22:21:49 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Apr 4 22:21:49 2016 -0700
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/StaleIndexCleanupTest.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6cfda8a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 27d969c..20fe4e9 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -70,6 +70,7 @@ import static org.junit.Assert.assertTrue;
* Test on read style clean-up of stale ElasticSearch indexes.
*/
@NotThreadSafe
+@Ignore("These can be re-enabled when they properly know when Usergrid is set to de-index old versions of entities or not")
public class StaleIndexCleanupTest extends AbstractCoreIT {
private static final Logger logger = LoggerFactory.getLogger( StaleIndexCleanupTest.class );
public static final String EVENTS_DISABLED = "corepersistence.events.disabled";
@@ -133,7 +134,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
-
+
/**
* Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an
* entity is deleted its old indexes are cleared from ElasticSearch.
[29/50] usergrid git commit: Concurreny / threading changes. Remove
Quorum reads that aren't necessarily consistency related problems.
Posted by sn...@apache.org.
Concurreny / threading changes. Remove Quorum reads that aren't necessarily consistency related problems.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/794bbd44
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/794bbd44
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/794bbd44
Branch: refs/heads/master
Commit: 794bbd44c9a546b11cdc9729d15ffcf24c26662d
Parents: 8c0338c
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Apr 13 18:53:09 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Apr 13 18:55:00 2016 +0200
----------------------------------------------------------------------
.../asyncevents/AsyncEventServiceImpl.java | 2 +-
.../corepersistence/index/IndexProcessorFig.java | 2 +-
.../corepersistence/service/ServiceSchedulerFig.java | 2 +-
.../collection/mvcc/stage/write/WriteUniqueVerify.java | 9 +++++++--
.../collection/serialization/SerializationFig.java | 3 +++
.../persistence/core/executor/TaskExecutorFactory.java | 10 +++++++---
.../apache/usergrid/persistence/queue/QueueFig.java | 4 ++++
.../services/notifications/gcm/GCMAdapter.java | 13 ++++++++-----
tests/integration/test/notifications/notifications.js | 5 -----
9 files changed, 32 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/794bbd44/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index d180919..82ad5be 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -493,7 +493,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
if(message == null) {
// provide some time back pressure before performing a quorum read
- if ( System.currentTimeMillis() > elasticsearchIndexEvent.getCreationTime() + queueFig.getLocalQuorumTimeout() ) {
+ if ( queueFig.getQuorumFallback() && System.currentTimeMillis() > elasticsearchIndexEvent.getCreationTime() + queueFig.getLocalQuorumTimeout() ) {
if(logger.isDebugEnabled()){
logger.debug("ES batch with id {} not found, reading with strong consistency", messageId);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/794bbd44/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index b94da65..c05c047 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -77,7 +77,7 @@ public interface IndexProcessorFig extends GuicyFig {
/**
* The number of worker threads used to read index write requests from the queue.
*/
- @Default("16")
+ @Default("8")
@Key(ELASTICSEARCH_WORKER_COUNT)
int getWorkerCount();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/794bbd44/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ServiceSchedulerFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ServiceSchedulerFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ServiceSchedulerFig.java
index e585ee3..764bba1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ServiceSchedulerFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ServiceSchedulerFig.java
@@ -50,7 +50,7 @@ public interface ServiceSchedulerFig extends GuicyFig {
- @Default("100")
+ @Default("50")
@Key( SERVICE_IMPORT_THREADS)
int getImportThreadPoolSize();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/794bbd44/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 780b83b..d05f838 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -66,6 +66,8 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
private final UniqueValueSerializationStrategy uniqueValueStrat;
+ public static int uniqueVerifyPoolSize = 100;
+
protected final SerializationFig serializationFig;
protected final Keyspace keyspace;
@@ -83,6 +85,8 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
this.uniqueValueStrat = uniqueValueSerializiationStrategy;
this.serializationFig = serializationFig;
+
+ uniqueVerifyPoolSize = this.serializationFig.getUniqueVerifyPoolSize();
}
@@ -175,7 +179,8 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
@Override
protected Map<String, Field> getFallback() {
- return executeStrategy(fig.getConsistentReadCL());
+ // fallback with same CL as there are many reasons the 1st execution failed, not just due to consistency problems
+ return executeStrategy(fig.getReadCL());
}
public Map<String, Field> executeStrategy(ConsistencyLevel consistencyLevel){
@@ -219,5 +224,5 @@ public class WriteUniqueVerify implements Action1<CollectionIoEvent<MvccEntity>>
public static final HystrixCommand.Setter
REPLAY_GROUP = HystrixCommand.Setter.withGroupKey(
HystrixCommandGroupKey.Factory.asKey( "uniqueVerify" ) ).andThreadPoolPropertiesDefaults(
- HystrixThreadPoolProperties.Setter().withCoreSize( 100 ) );
+ HystrixThreadPoolProperties.Setter().withCoreSize( uniqueVerifyPoolSize ) );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/794bbd44/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index ca9cd99..96759ba 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -54,4 +54,7 @@ public interface SerializationFig extends GuicyFig {
@Default( "5000000" )
int getMaxEntitySize();
+ @Key ( "usergrid.uniqueverify.poolsize" )
+ @Default( "150" )
+ int getUniqueVerifyPoolSize();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/794bbd44/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/executor/TaskExecutorFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/executor/TaskExecutorFactory.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/executor/TaskExecutorFactory.java
index 5e7761c..c1c6207 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/executor/TaskExecutorFactory.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/executor/TaskExecutorFactory.java
@@ -168,8 +168,10 @@ public class TaskExecutorFactory {
@Override
public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
- logger.warn( "{} task queue full, rejecting task {} and running in thread {}", poolName, r,
- Thread.currentThread().getName() );
+ if(logger.isDebugEnabled()) {
+ logger.debug("{} task queue full, rejecting task {} and running in thread {}", poolName, r,
+ Thread.currentThread().getName());
+ }
//We've decided we want to have a "caller runs" policy, to just invoke the task when rejected
@@ -191,7 +193,9 @@ public class TaskExecutorFactory {
@Override
public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
- logger.warn( "{} task queue full, dropping task {}", poolName, r );
+ if(logger.isDebugEnabled()) {
+ logger.warn("{} task queue full, dropping task {}", poolName, r);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/794bbd44/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
index ca6e011..533314b 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
@@ -99,4 +99,8 @@ public interface QueueFig extends GuicyFig {
@Default( "3000" ) // 3 seconds
int getQueuePollTimeshift();
+ @Key( "usergrid.queue.quorum.fallback")
+ @Default("false") // 30 seconds
+ boolean getQuorumFallback();
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/794bbd44/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
index e334a54..af0bc78 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
@@ -242,11 +242,14 @@ public class GCMAdapter implements ProviderAdapter {
payload.remove(priorityKey);
}
-
- // add our source notification payload data into the Message Builder
- // Message.Builder requires the payload to be Map<String,String> so blindly cast
- Map<String,String> dataMap = (Map<String,String>) payload;
- dataMap.forEach( (key, value) -> builder.addData(key, value));
+//
+// // add our source notification payload data into the Message Builder
+// // Message.Builder requires the payload to be Map<String,String> so blindly cast
+// Map<String,String> dataMap = (Map<String,String>) payload;
+//
+// dataMap.forEach( (key, value) -> builder.addData(key, value));
+
+ builder.addData("data", JSON.toString(payload));
Message message = builder.build();
MulticastResult multicastResult;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/794bbd44/tests/integration/test/notifications/notifications.js
----------------------------------------------------------------------
diff --git a/tests/integration/test/notifications/notifications.js b/tests/integration/test/notifications/notifications.js
index 644510f..7a255e5 100644
--- a/tests/integration/test/notifications/notifications.js
+++ b/tests/integration/test/notifications/notifications.js
@@ -278,7 +278,6 @@ module.exports = {
function (err, notification) {
should(err).be.null;
notification.should.not.be.null;
- notification.expectedCount.should.be.equal(1);
setTimeout(function() {
// wait a second before proceeding
@@ -306,7 +305,6 @@ module.exports = {
function (err, notification) {
should(err).be.null;
notification.should.not.be.null;
- notification.expectedCount.should.be.equal(1);
setTimeout(function() {
// wait a second before proceeding
@@ -334,9 +332,6 @@ module.exports = {
should(err).be.null;
notification.should.not.be.null;
- // we set up 2 groups of the same 5 users. if duplicate filtering is working,
- // we'll only have 5 expected
- notification.expectedCount.should.be.equal(5);
setTimeout(function() {
// wait a second before proceeding
[33/50] usergrid git commit: This closes #501. Merge branch 'pr/501'
into release-2.1.1
Posted by sn...@apache.org.
This closes #501. Merge branch 'pr/501' into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2e576f37
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2e576f37
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2e576f37
Branch: refs/heads/master
Commit: 2e576f3737689d45b7ea7aa9fdd137138eea0702
Parents: 10ff27c 3a442b0
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Apr 13 17:16:35 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Apr 13 17:16:35 2016 -0400
----------------------------------------------------------------------
.../collection/CollectionsResourceIT.java | 40 ++++++++++++++++++++
.../rest/test/resource/ClientSetup.java | 10 +++++
.../services/AbstractCollectionService.java | 23 +++++++++--
3 files changed, 70 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[17/50] usergrid git commit: First pass to allow more than 1k
notifications to be sent for groups/queries targeting users and devices.
Posted by sn...@apache.org.
First pass to allow more than 1k notifications to be sent for groups/queries targeting users and devices.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8e4d7eef
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8e4d7eef
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8e4d7eef
Branch: refs/heads/master
Commit: 8e4d7eef2ca0967491e9ef863c78880e002575d1
Parents: 74de4bc
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Apr 10 23:20:10 2016 +0300
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Apr 10 23:20:10 2016 +0300
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 34 ++++---
.../services/notifications/TaskManager.java | 7 +-
.../impl/ApplicationQueueManagerImpl.java | 100 ++++++++++++++++---
3 files changed, 113 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8e4d7eef/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 67e92f8..b5a4107 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -17,14 +17,7 @@
package org.apache.usergrid.corepersistence;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -318,14 +311,31 @@ public class CpRelationManager implements RelationManager {
final String ql;
- if ( startResult != null ) {
- ql = "select * where created > " + startResult.timestamp();
- }
- else {
+
+ if (startResult != null ) {
+
+ // UUID timestamp is a different measure than 'created' field on entities
+ Calendar uuidEpoch = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+ uuidEpoch.clear();
+ uuidEpoch.set(1582, 9, 15, 0, 0, 0); // 9 = October
+ long epochMillis = uuidEpoch.getTime().getTime();
+
+ long time = (startResult.timestamp() / 10000L) + epochMillis;
+
+ if ( !reversed ) {
+ ql = "select * where created > " + time;
+ } else {
+ ql = "select * where created < " + time;
+ }
+
+ } else {
ql = "select *";
}
Query query = Query.fromQL( ql );
+ if(query == null ){
+ throw new RuntimeException("Unable to get data for collection: "+collectionName);
+ }
query.setLimit( count );
query.setReversed( reversed );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8e4d7eef/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
index 3e78210..954724f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
@@ -131,13 +131,14 @@ public class TaskManager {
}
if ( debug || hasError) {
+
+ List<EntityRef> entities = Arrays.asList(notification, device);
+
if (receipt.getUuid() == null) {
Receipt savedReceipt = em.create(receipt);
- receipt.setUuid(savedReceipt.getUuid());
- List<EntityRef> entities = Arrays.asList(notification, device);
em.addToCollections(entities, Notification.RECEIPTS_COLLECTION, savedReceipt);
} else {
- em.update(receipt);
+ em.addToCollections(entities, Notification.RECEIPTS_COLLECTION, receipt);
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8e4d7eef/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 12a47b6..04e60b7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -34,6 +34,7 @@ import rx.Observable;
import rx.Subscriber;
import rx.functions.Func1;
+import java.io.IOException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@@ -119,7 +120,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
final UUID appId = em.getApplication().getUuid();
final Map<String, Object> payloads = notification.getPayloads();
- final Func1<EntityRef, EntityRef> sendMessageFunction = deviceRef -> {
+ final Func1<EntityRef, ApplicationQueueMessage> sendMessageFunction = deviceRef -> {
try {
long now = System.currentTimeMillis();
@@ -143,7 +144,8 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}
if (notifierId == null) {
- return deviceRef;
+ //TODO need to leverage optional here
+ //return deviceRef;
}
ApplicationQueueMessage message = new ApplicationQueueMessage(appId, notification.getUuid(), deviceRef.getUuid(), notifierKey, notifierId);
@@ -153,16 +155,19 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
notification.setQueued(System.currentTimeMillis());
}
- qm.sendMessage(message);
deviceCount.incrementAndGet();
- queueMeter.mark();
+
+ return message;
} catch (Exception deviceLoopException) {
logger.error("Failed to add device", deviceLoopException);
errorMessages.add("Failed to add device: " + deviceRef.getUuid() + ", error:" + deviceLoopException);
+
+ //TODO need an optional here
+ return new ApplicationQueueMessage(appId, notification.getUuid(), deviceRef.getUuid(), "test", "test");
}
- return deviceRef;
+
};
@@ -174,9 +179,28 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}, 10)
.distinct(ref -> ref.getUuid())
.map(sendMessageFunction)
+ .buffer(100)
+ .doOnNext( applicationQueueMessages -> {
+
+ applicationQueueMessages.forEach( message -> {
+
+ try {
+
+ qm.sendMessage( message );
+ queueMeter.mark();
+
+ } catch (IOException e) {
+ logger.error("Unable to queue notification for notification UUID {} and device UUID {} ",
+ message.getNotificationId(), message.getDeviceId());
+ }
+
+ });
+
+
+ })
.doOnError(throwable -> logger.error("Failed while trying to send notification", throwable));
- processMessagesObservable.toBlocking().lastOrDefault(null);
+ processMessagesObservable.toBlocking(); // let this run and block the async thread, messages are queued
}
@@ -487,20 +511,70 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
private List<EntityRef> getDevices(EntityRef ref) {
- List<EntityRef> devices = Collections.EMPTY_LIST;
+ List<EntityRef> devices = new ArrayList<>();
+
try {
+
if ("device".equals(ref.getType())) {
+
devices = Collections.singletonList(ref);
+
} else if ("user".equals(ref.getType())) {
- devices = em.getCollection(ref, "devices", null, Query.MAX_LIMIT,
- Query.Level.REFS, false).getRefs();
+
+ UUID start = null;
+ boolean initial = true;
+ int resultSize = 0;
+ while( initial || resultSize >= Query.DEFAULT_LIMIT) {
+
+ initial = false;
+
+ final List<EntityRef> mydevices = em.getCollection(ref, "devices", start, Query.DEFAULT_LIMIT,
+ Query.Level.REFS, true).getRefs();
+
+ resultSize = mydevices.size();
+ if(mydevices.size() > 0){
+ start = mydevices.get(mydevices.size() - 1 ).getUuid();
+ }
+
+
+ devices.addAll( mydevices );
+
+
+ }
+
} else if ("group".equals(ref.getType())) {
- devices = new ArrayList<>();
- for (EntityRef r : em.getCollection(ref, "users", null,
- Query.MAX_LIMIT, Query.Level.REFS, false).getRefs()) {
- devices.addAll(getDevices(r));
+
+ //devices = new ArrayList<>();
+ UUID start = null;
+ boolean initial = true;
+ int resultSize = 0;
+
+ while( initial || resultSize >= Query.DEFAULT_LIMIT){
+
+ initial = false;
+ final List<EntityRef> myusers = em.getCollection(ref, "users", start,
+ Query.DEFAULT_LIMIT, Query.Level.REFS, true).getRefs();
+
+ resultSize = myusers.size();
+ if(myusers.size() > 0){
+ start = myusers.get(myusers.size() - 1 ).getUuid();
+ }
+
+
+ // don't allow a single user to have more than 100 devices?
+ for (EntityRef user : myusers) {
+
+ devices.addAll( em.getCollection(user, "devices", null, 100,
+ Query.Level.REFS, true).getRefs() );
+
+
+ }
+
}
+
+
+
}
} catch (Exception e) {
[24/50] usergrid git commit: Forcing buffer to only emit batches that
contain items.
Posted by sn...@apache.org.
Forcing buffer to only emit batches that contain items.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/19892a9d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/19892a9d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/19892a9d
Branch: refs/heads/master
Commit: 19892a9df867733737b5d06f8f853a5529acffaf
Parents: 6b9e5ef
Author: George Reyes <gr...@apache.org>
Authored: Tue Apr 12 20:35:48 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Tue Apr 12 20:35:48 2016 -0700
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/index/IndexServiceImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/19892a9d/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index f3f48cc..ad997c8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -131,7 +131,7 @@ public class IndexServiceImpl implements IndexService {
//try to send a whole batch if we can
final Observable<IndexOperationMessage> batches = sourceEdgesToIndex
- .buffer(250, TimeUnit.MILLISECONDS, indexFig.getIndexBatchSize() )
+ .buffer(indexFig.getIndexBatchSize() )
//map into batches based on our buffer size
.flatMap( buffer -> Observable.from( buffer )
[49/50] usergrid git commit: Update to newer version of GCM server
which lets Usergrid directly set the data object within the SDK instead of
just adding key/value pairs.
Posted by sn...@apache.org.
Update to newer version of GCM server which lets Usergrid directly set the data object within the SDK instead of just adding key/value pairs.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d0691856
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d0691856
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d0691856
Branch: refs/heads/master
Commit: d06918562a6e3f76a3ca98d384d1c5ef6beb1c23
Parents: b8f502f
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 19 17:37:41 2016 +0100
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 19 17:37:41 2016 +0100
----------------------------------------------------------------------
stack/pom.xml | 2 +-
.../services/notifications/gcm/GCMAdapter.java | 12 +++-----
.../gcm/NotificationsServiceIT.java | 32 ++++++++++++++++++++
3 files changed, 37 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d0691856/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index f052bf9..a94233e 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -121,7 +121,7 @@
<antlr.version>3.4</antlr.version>
<tika.version>1.4</tika.version>
<mockito.version>1.10.8</mockito.version>
- <io.apigee.gcm.version>1.0.0</io.apigee.gcm.version>
+ <io.apigee.gcm.version>1.0.1</io.apigee.gcm.version>
<!-- only use half the cores on the machine for testing -->
<usergrid.it.parallel>methods</usergrid.it.parallel>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d0691856/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
index af0bc78..7929ad4 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/gcm/GCMAdapter.java
@@ -242,14 +242,10 @@ public class GCMAdapter implements ProviderAdapter {
payload.remove(priorityKey);
}
-//
-// // add our source notification payload data into the Message Builder
-// // Message.Builder requires the payload to be Map<String,String> so blindly cast
-// Map<String,String> dataMap = (Map<String,String>) payload;
-//
-// dataMap.forEach( (key, value) -> builder.addData(key, value));
-
- builder.addData("data", JSON.toString(payload));
+
+ builder.setData(payload);
+ // GCM will accept Map<String,Object> but builder.build().toString() will throw a class cast
+ // exception, but luckily Message.toString() is not used anywhere in the GCM SDK or Usergrid
Message message = builder.build();
MulticastResult multicastResult;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d0691856/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 8782fe3..77abb56 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -167,6 +167,38 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
}
@Test
+ public void singlePushNotificationMapPayload() throws Exception {
+
+ app.clear();
+ Map<String, Object> topLevel = new HashMap<>();
+ Map<String, String> mapPayload = new HashMap<String, String>(){{
+ put("key1", "value1");
+ put("key2", "value2");
+
+ }};
+ topLevel.put("enabler", mapPayload);
+ Map<String, Object> payloads = new HashMap<>(1);
+ payloads.put(notifier.getUuid().toString(), topLevel);
+ app.put("payloads", payloads);
+ app.put("queued", System.currentTimeMillis());
+ app.put("debug", true);
+ app.put("expire", System.currentTimeMillis() + 300000); // add 5 minutes to current time
+
+ Entity e = app.testRequest(ServiceAction.POST, 1, "devices", device1.getUuid(), "notifications").getEntity();
+ app.testRequest(ServiceAction.GET, 1, "notifications", e.getUuid());
+
+ Notification notification = app.getEntityManager().get(e.getUuid(), Notification.class);
+
+ //assertEquals(
+ // notification.getPayloads().get(notifier.getUuid().toString()),
+ // payload);
+
+ // perform push //
+ notification = notificationWaitForComplete(notification);
+ checkReceipts(notification, 1);
+ }
+
+ @Test
public void singlePushNotificationNoReceipts() throws Exception {
app.clear();
[15/50] usergrid git commit: Merge branch
'selectiveIndexingCherryPicked' into release-2.1.1
Posted by sn...@apache.org.
Merge branch 'selectiveIndexingCherryPicked' into release-2.1.1
* selectiveIndexingCherryPicked: (33 commits)
Removed all instance of java.util.Optional from selective indexing.
Add some more comments and clarity to selective indexing.
Added tests and code to cover the following cases: Somebody doesn't have a field array in the payload submitted to the index schema. Somebody submits something other than a JSON array to the "fields" field.
Cleaned up the old code that used to do the filtering.
Created a easier to understand algorithm that checks for top level indexing values. Passes existing tests.
Forced the indexing schema to store the map instead of the JSON string that way we only do the parsing once if we're doing a lot of get's on the schema.
Added a way to use the existing reindex code to reindex the collection rather than copying and pasting the reindex code.
Added a delete test and a delete endpoint so that we can delete schema's
Added a put that will automatically invalidate the cache.
Removed the index schema cache from the manager cache as it can and should stand alone.
Added the index schema cache to the index service.
Added first instances of using a cache for access index schema values
Removed all unused imports in cpRelationManager and IndexServiceImpl and EsEntityIndexBatchImpl. Re-added final keyword to indexing interface
Changed the seemingly not working delay to 50 millisecond , moved the collection reindexing timer updating code to be easier to follow.
Cleaning up the left over code from the endeavor.
Simplified the CollectionResource and remove recursive call based on serviceResource wonkiness
Working lower level selective indexing. Working field updates when we do reindexing and stuff. Added test proving that it works.
Comments with changes to make to selective indexing. Changes to reindexing to add a delay given the reindexing call.
Commented out the delay because it broke tests. Cleaned up some commented out code and spaces.
Cleaned up the updating test to verify that a collection of 10 will be indexed.
...
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/74de4bca
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/74de4bca
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/74de4bca
Branch: refs/heads/master
Commit: 74de4bca3a6c74b1a8491b9bc093cda9e9b26add
Parents: 29d7660 d751ac1
Author: George Reyes <gr...@apache.org>
Authored: Fri Apr 8 13:20:27 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Fri Apr 8 13:20:27 2016 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 -
.../usergrid/corepersistence/CoreModule.java | 6 +
.../corepersistence/CpEntityManager.java | 111 +++-
.../corepersistence/CpEntityManagerFactory.java | 8 +-
.../corepersistence/CpManagerCache.java | 2 +-
.../corepersistence/CpRelationManager.java | 5 -
.../usergrid/corepersistence/ManagerCache.java | 2 +-
.../asyncevents/EventBuilderImpl.java | 1 -
.../corepersistence/index/IndexSchemaCache.java | 50 ++
.../index/IndexSchemaCacheFactory.java | 44 ++
.../index/IndexSchemaCacheFig.java | 39 ++
.../index/IndexSchemaCacheImpl.java | 119 ++++
.../corepersistence/index/IndexServiceImpl.java | 94 ++-
.../index/ReIndexRequestBuilder.java | 7 +
.../index/ReIndexRequestBuilderImpl.java | 32 +
.../index/ReIndexServiceImpl.java | 39 +-
.../usergrid/persistence/EntityManager.java | 7 +
.../persistence/cassandra/ApplicationCF.java | 2 +-
.../impl/EntityCollectionManagerImpl.java | 2 +-
.../core/consistency/TimeServiceImpl.java | 5 +-
.../persistence/map/impl/MapManagerImpl.java | 6 +
.../persistence/index/EntityIndexBatch.java | 13 +-
.../index/impl/EntityToMapConverter.java | 93 ++-
.../index/impl/EsEntityIndexBatchImpl.java | 21 +-
.../persistence/index/impl/IndexOperation.java | 9 +-
.../persistence/index/impl/EntityIndexTest.java | 2 +-
.../org/apache/usergrid/rest/ApiResponse.java | 7 +-
.../rest/applications/ApplicationResource.java | 2 +-
.../rest/applications/CollectionResource.java | 233 +++++++
.../rest/applications/ServiceResource.java | 111 +++-
.../usergrid/rest/system/IndexResource.java | 6 +
.../collection/CollectionsResourceIT.java | 622 ++++++++++++++++++-
.../test/resource/endpoints/NamedResource.java | 22 +
.../services/AbstractCollectionService.java | 39 ++
.../services/AbstractConnectionsService.java | 13 +
.../usergrid/services/AbstractService.java | 11 +-
.../org/apache/usergrid/services/Service.java | 3 +
.../usergrid/services/ServiceManager.java | 1 -
.../usergrid/services/ServiceRequest.java | 5 +
.../applications/ApplicationsService.java | 14 +
40 files changed, 1738 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/74de4bca/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
[03/50] usergrid git commit: Revert "Ignore non critical tests for
now."
Posted by sn...@apache.org.
Revert "Ignore non critical tests for now."
This reverts commit a6cfda8ae0ca9dabcd6d18a966eb9848d91f2749.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fbb5807e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fbb5807e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fbb5807e
Branch: refs/heads/master
Commit: fbb5807e46a6270eadfc3822a60112d4a9e3faf1
Parents: d3df239
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 5 13:47:11 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 5 13:47:11 2016 -0700
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/StaleIndexCleanupTest.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbb5807e/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 20fe4e9..27d969c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -70,7 +70,6 @@ import static org.junit.Assert.assertTrue;
* Test on read style clean-up of stale ElasticSearch indexes.
*/
@NotThreadSafe
-@Ignore("These can be re-enabled when they properly know when Usergrid is set to de-index old versions of entities or not")
public class StaleIndexCleanupTest extends AbstractCoreIT {
private static final Logger logger = LoggerFactory.getLogger( StaleIndexCleanupTest.class );
public static final String EVENTS_DISABLED = "corepersistence.events.disabled";
@@ -134,7 +133,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
-
+
/**
* Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an
* entity is deleted its old indexes are cleared from ElasticSearch.
[05/50] usergrid git commit: Revert "Ignore broken test for now."
Posted by sn...@apache.org.
Revert "Ignore broken test for now."
This reverts commit 0cb4e08f2c44a937156823b0ee657300ad9686bb.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a976cffa
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a976cffa
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a976cffa
Branch: refs/heads/master
Commit: a976cffadc8f327b1cd2f08124aaaeefc415a11b
Parents: c9d532f
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 5 13:47:26 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 5 13:47:26 2016 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a976cffa/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 27d969c..536f9f5 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -243,7 +243,6 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
/**
* Test that the AbstractElasticsearchFilter de-indexes old versions when reading candidates
*/
- @Ignore
@Test()
public void testCleanupOnUpdate() throws Exception {
[43/50] usergrid git commit: Introduce a graph iterator for
segmenting push notifications.
Posted by sn...@apache.org.
Introduce a graph iterator for segmenting push notifications.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/32ab5da0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/32ab5da0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/32ab5da0
Branch: refs/heads/master
Commit: 32ab5da0af66d2dfc4ed9fdb8ddf84a6b0231c03
Parents: f064c49
Author: Michael Russo <mr...@apigee.com>
Authored: Sat Apr 16 00:08:32 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Sat Apr 16 00:08:32 2016 +0200
----------------------------------------------------------------------
.../persistence/NotificationGraphIterator.java | 119 +++++++++++++++++++
.../apache/usergrid/persistence/PathQuery.java | 35 ++++--
.../persistence/entities/Notification.java | 62 +++++++++-
.../notifications/NotificationDeviceFilter.java | 45 +++++++
.../notifications/NotificationsService.java | 6 +
.../impl/ApplicationQueueManagerImpl.java | 101 ++++++++++++++--
6 files changed, 352 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32ab5da0/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
new file mode 100644
index 0000000..b83f555
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+
+public class NotificationGraphIterator implements ResultsIterator, Iterable {
+
+ private static final Logger logger = LoggerFactory.getLogger(NotificationGraphIterator.class);
+
+
+ EntityManager entityManager;
+
+ private Iterator<EntityRef> source;
+ private Query query;
+ private Iterator currentIterator;
+
+
+ public NotificationGraphIterator(EntityManager entityManager,
+ Iterator<EntityRef> source,
+ Query query) {
+
+ this.entityManager = entityManager;
+ this.source = source;
+ this.query = query;
+
+ }
+
+ @Override
+ public Iterator iterator() {
+ return this;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (source == null) {
+ return false;
+ }
+ if (currentIterator != null && currentIterator.hasNext()) {
+ return true;
+ }
+ while (source.hasNext()) {
+ EntityRef ref = source.next();
+ Results r = getResultsFor(ref);
+ if (r.size() > 0) {
+ currentIterator = new PagingResultsIterator(r, query.getResultsLevel());
+ return currentIterator.hasNext();
+ }
+ }
+ currentIterator = null;
+ source = null;
+ return false;
+ }
+
+
+ @Override
+ public Object next() {
+
+
+
+
+ return (currentIterator != null) ? currentIterator.next() : null;
+ }
+
+ @Override
+ public boolean hasPages() {
+ return currentIterator != null && currentIterator instanceof ResultsIterator && ((ResultsIterator) currentIterator).hasPages();
+ }
+
+
+ private Results getResultsFor(EntityRef ref) {
+
+ try {
+
+ if (query.getCollection() != null) {
+
+ if(logger.isTraceEnabled()) {
+ logger.trace("Fetching with refType: {}, collection: {} with no query",
+ ref.getType(), query.getCollection());
+ }
+ return entityManager.searchCollection(ref, query.getCollection(), null);
+
+ } else {
+
+ if(logger.isTraceEnabled()) {
+ logger.trace("Searching target entities with refType: {} for collection: {} with no query",
+ ref.getType(), query.getCollection());
+ }
+
+ query.setQl("select *");
+ return entityManager.searchTargetEntities(ref, query);
+
+ }
+
+
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32ab5da0/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
index 55839a6..215f6ac 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
@@ -22,7 +22,6 @@ import java.util.Iterator;
import java.util.UUID;
import org.apache.usergrid.persistence.Query.Level;
-import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.utils.InflectionUtils;
@@ -52,11 +51,10 @@ public class PathQuery<E> {
/**
* top level
- *
- * @param head the top-level entity
+ * @param head the top-level entity
* @param query the query - must have a collection or connectType value set
*/
- public PathQuery( EntityRef head, Query query ) {
+ public PathQuery(EntityRef head, Query query) {
if ( query.getCollection() == null && query.getConnectionType() == null ) {
throw new IllegalArgumentException( "Query must have a collection or connectionType value" );
}
@@ -93,7 +91,7 @@ public class PathQuery<E> {
return new PagingResultsIterator( getHeadResults( em ), query.getResultsLevel() );
}
else {
- return new MultiQueryIterator( em, source.refIterator( em ), query );
+ return new MultiQueryIterator( em, source.refIterator( em, false), query );
}
}
catch ( Exception e ) {
@@ -101,6 +99,24 @@ public class PathQuery<E> {
}
}
+ public Iterator<E> graphIterator( EntityManager em ) {
+ try {
+
+ if ( uuid != null && type != null ) {
+ return new PagingResultsIterator( getHeadResults( em ), query.getResultsLevel() );
+ }else {
+
+ return new NotificationGraphIterator(em, source.refIterator(em, true), query);
+ }
+
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( e );
+ }
+ }
+
+
+
protected Results getHeadResults( EntityManager em ) throws Exception {
@@ -123,7 +139,7 @@ public class PathQuery<E> {
}
- protected Iterator refIterator( EntityManager em ) throws Exception {
+ protected Iterator refIterator(EntityManager em, boolean useGraph) throws Exception {
if ( query.getQl() == null && query.getSingleNameOrEmailIdentifier() != null){
@@ -140,7 +156,12 @@ public class PathQuery<E> {
q = new Query( q );
q.setResultsLevel( Level.REFS );
}
- return new MultiQueryIterator( em, source.refIterator( em ), q );
+ if( useGraph){
+ return new NotificationGraphIterator( em, source.refIterator( em, true), q );
+ }else{
+ return new MultiQueryIterator( em, source.refIterator( em, false ), q );
+
+ }
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32ab5da0/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
index 6a6e3fa..aca10cf 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
@@ -26,6 +26,8 @@ import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.annotations.EntityCollection;
import org.apache.usergrid.persistence.annotations.EntityProperty;
import org.apache.usergrid.persistence.index.query.Identifier;
+import org.apache.usergrid.utils.InflectionUtils;
+
/**
* The entity class for representing Notifications.
@@ -77,6 +79,10 @@ public class Notification extends TypedEntity {
@EntityProperty
protected Long expire;
+ /** Stores the number of devices processed */
+ @EntityProperty
+ protected int deviceProcessedCount;
+
/** True if notification is canceled */
@EntityProperty
protected Boolean canceled;
@@ -89,6 +95,10 @@ public class Notification extends TypedEntity {
@EntityProperty
protected String priority;
+ /** Flag to signal Usergrid to use graph traversal + filtering to find devices */
+ @EntityProperty
+ protected boolean useGraph;
+
/** Error messages that may have been encountered by Usergrid when trying to process the notification */
@EntityProperty
protected String errorMessage;
@@ -104,6 +114,9 @@ public class Notification extends TypedEntity {
@EntityProperty
protected Map<String, Long> statistics;
+ @EntityProperty
+ protected Map<String, Object> filters;
+
public Notification() {
pathQuery = new PathTokens();
@@ -173,6 +186,15 @@ public class Notification extends TypedEntity {
}
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ public int getDeviceProcessedCount() {
+ return deviceProcessedCount;
+ }
+
+ public void setDeviceProcessedCount(int deviceProcessedCount) {
+ this.deviceProcessedCount = deviceProcessedCount;
+ }
+
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public Boolean getCanceled() {
return canceled;
}
@@ -191,6 +213,15 @@ public class Notification extends TypedEntity {
}
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ public boolean getUseGraph() {
+ return useGraph;
+ }
+
+ public void setUseGraph(boolean useGraph) {
+ this.useGraph = useGraph;
+ }
+
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public boolean getDebug() {
return debug;
}
@@ -252,6 +283,15 @@ public class Notification extends TypedEntity {
this.statistics = statistics;
}
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ public Map<String, Object> getFilters() {
+ return filters;
+ }
+
+ public void setFilters(Map<String, Object> filters) {
+ this.filters = filters;
+ }
+
public void updateStatistics(long sent, long errors) {
if (this.statistics == null) {
this.statistics = new HashMap<String, Long>(2);
@@ -341,7 +381,8 @@ public class Notification extends TypedEntity {
@JsonIgnore
public PathQuery<Device> buildPathQuery() {
PathQuery pathQuery = null;
- for (PathToken pathToken : getPathTokens()) {
+ List<PathToken> pathTokens = getPathTokens();
+ for (PathToken pathToken : pathTokens) {
String collection = pathToken.getCollection();
Query query = new Query();
if(pathToken.getQl() != null){
@@ -365,6 +406,25 @@ public class Notification extends TypedEntity {
if (pathQuery == null) {
pathQuery = new PathQuery(getApplicationRef(), query);
+
+ if ( pathTokens.size() == 1 && collection.equals(InflectionUtils.pluralize(Group.ENTITY_TYPE) )){
+
+ Query usersQuery = new Query();
+ usersQuery.setQl("select *");
+ usersQuery.setCollection("users");
+ usersQuery.setLimit(100);
+
+ Query devicesQuery = new Query();
+ devicesQuery.setQl("select *");
+ devicesQuery.setCollection("devices");
+ usersQuery.setLimit(100);
+
+
+ // build up the chain so the proper iterators can be used later
+ pathQuery = pathQuery.chain( usersQuery ).chain( devicesQuery );
+
+ }
+
} else {
pathQuery = pathQuery.chain(query);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32ab5da0/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationDeviceFilter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationDeviceFilter.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationDeviceFilter.java
new file mode 100644
index 0000000..35700ea
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationDeviceFilter.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.services.notifications;
+
+
+public class NotificationDeviceFilter {
+
+
+ private String key;
+ private Object value;
+
+ public NotificationDeviceFilter(String key, Object value){
+
+ this.key = key;
+ this.value = value;
+
+ }
+
+ public Object getValue(){
+
+ return this.value;
+
+ }
+
+ public String getKey(){
+
+ return this.key;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32ab5da0/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
index 05c1cd7..50eb883 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.services.notifications;
import java.util.*;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -84,6 +85,7 @@ public class NotificationsService extends AbstractCollectionService {
private ServiceManagerFactory smf;
private EntityManagerFactory emf;
private QueueManagerFactory queueManagerFactory;
+ private EntityCollectionManagerFactory ecmf;
public NotificationsService() {
if (logger.isTraceEnabled()) {
@@ -139,9 +141,13 @@ public class NotificationsService extends AbstractCollectionService {
try {
validate(null, context.getPayload());
Notification.PathTokens pathTokens = getPathTokens(context.getRequest().getOriginalParameters());
+
// default saving of receipts
+ context.getProperties().put("filters", context.getProperties().getOrDefault("filters", new HashMap<>()));
+ context.getProperties().put("useGraph", context.getProperties().getOrDefault("useGraph", false));
context.getProperties().put("saveReceipts", context.getProperties().getOrDefault("saveReceipts", true));
context.getProperties().put("processingFinished", 0L); // defaulting processing finished to 0
+ context.getProperties().put("deviceProcessedCount", 0); // defaulting processing finished to 0
context.getProperties().put("state", Notification.State.CREATED);
context.getProperties().put("pathQuery", pathTokens);
context.setOwner(sm.getApplication());
http://git-wip-us.apache.org/repos/asf/usergrid/blob/32ab5da0/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 2466164..487ea1f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -19,7 +19,10 @@ package org.apache.usergrid.services.notifications.impl;
import com.codahale.metrics.Meter;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.*;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.queue.QueueManager;
@@ -97,14 +100,21 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
final AtomicInteger deviceCount = new AtomicInteger(); //count devices so you can make a judgement on batching
final ConcurrentLinkedQueue<String> errorMessages = new ConcurrentLinkedQueue<>(); //build up list of issues
-
//get devices in querystring, and make sure you have access
if (pathQuery != null) {
final HashMap<Object, ProviderAdapter> notifierMap = getAdapterMap();
if (logger.isTraceEnabled()) {
logger.trace("notification {} start query", notification.getUuid());
}
- final Iterator<Device> iterator = pathQuery.iterator(em);
+
+
+ // the main iterator can use graph traversal or index querying
+ final Iterator<Device> iterator;
+ if( notification.getUseGraph()){
+ iterator = pathQuery.graphIterator(em);
+ }else{
+ iterator = pathQuery.iterator(em);
+ }
//if there are more pages (defined by PAGE_SIZE) you probably want this to be async, also if this is already a job then don't reschedule
if (iterator instanceof ResultsIterator && ((ResultsIterator) iterator).hasPages() && jobExecution == null) {
@@ -167,6 +177,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
};
+ final Map<String, Object> filters = notification.getFilters();
Observable processMessagesObservable = Observable.create(new IteratorObservable<Entity>(iterator))
.flatMap(entity -> {
@@ -180,10 +191,73 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
})
.distinct(ref -> ref.getUuid())
+ .flatMap( entityRef -> {
+
+ return Observable.just(entityRef).flatMap( ref -> {
+
+ if(logger.isTraceEnabled()){
+ logger.trace("Loading device: {}", ref.getUuid());
+
+ }
+ try {
+ return Observable.just(em.get(ref, Device.class));
+ }
+ catch (Exception e){
+
+ return Observable.empty();
+
+ }
+
+ }).subscribeOn(Schedulers.io());
+
+
+ }, 50)
+ .filter( device -> {
+
+ if(logger.isTraceEnabled()) {
+ logger.trace("Filtering device: {}", device.getUuid());
+ }
+
+
+ if(notification.getUseGraph() && filters.size() > 0 ) {
+
+ for (Map.Entry<String, Object> entry : filters.entrySet()) {
+
+ if ((device.getDynamicProperties().get(entry.getKey()) != null &&
+ device.getDynamicProperties().get(entry.getKey()).equals(entry.getValue())) ||
+
+ (device.getProperties().get(entry.getKey()) != null &&
+ device.getProperties().get(entry.getKey()).equals(entry.getValue()))
+
+ ) {
+
+
+ return true;
+ }
+
+ }
+
+ if(logger.isTraceEnabled()) {
+ logger.trace("Push notification filter matched for notification {}, so removing from notification",
+ device.getUuid(), notification.getUuid());
+ }
+ return false;
+
+
+ }
+
+ return true;
+
+ })
.map(sendMessageFunction)
.doOnNext( message -> {
try {
+
if(message.isPresent()){
+
+ if(logger.isTraceEnabled()) {
+ logger.trace("Queueing notification message for device: {}", message.get().getDeviceId());
+ }
qm.sendMessage( message.get() );
queueMeter.mark();
}
@@ -206,7 +280,10 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
try {
notification.setProcessingFinished(System.currentTimeMillis());
+ notification.setDeviceProcessedCount(deviceCount.get());
em.update(notification);
+ logger.info("{} devices processed for notification {}", deviceCount.get(), notification.getUuid());
+
} catch (Exception e) {
logger.error("Unable to set processing finished timestamp for notification");
}
@@ -569,9 +646,9 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
while( initial || resultSize >= LIMIT){
initial = false;
+
final List<EntityRef> myusers = em.getCollection(ref, "users", start,
LIMIT, Query.Level.REFS, true).getRefs();
-
resultSize = myusers.size();
if(myusers.size() > 0){
@@ -579,13 +656,21 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}
- // don't allow a single user to have more than 100 devices?
- for (EntityRef user : myusers) {
+ Observable.from(myusers).flatMap( user -> {
+
+ try {
+ devices.addAll(em.getCollection(user, "devices", null, 100,
+ Query.Level.REFS, true).getRefs());
+ }catch (Exception e){
+ logger.error ("Unable to fetch devices for user: {}", user.getUuid());
+ }
+ return Observable.from(Collections.singletonList(user));
+
+ }, 50).toBlocking().lastOrDefault(null);
+
+
- devices.addAll( em.getCollection(user, "devices", null, 100,
- Query.Level.REFS, true).getRefs() );
- }
}
[13/50] usergrid git commit: Merge branch '9f45130_lock_mgr_fix' into
release-2.1.1
Posted by sn...@apache.org.
Merge branch '9f45130_lock_mgr_fix' into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e0a99524
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e0a99524
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e0a99524
Branch: refs/heads/master
Commit: e0a995243a9edd7e7efd81922ffe36be1f14ea5b
Parents: 4326b10 b855728
Author: Mike Dunker <md...@apigee.com>
Authored: Fri Apr 8 08:55:16 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Fri Apr 8 08:55:16 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/batch/job/OnlyOnceJob.java | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0a99524/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
index 82e856d,ac56c7b..9b1fb05
--- a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
+++ b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
@@@ -58,9 -62,13 +62,13 @@@ public abstract class OnlyOnceJob imple
@PostConstruct
public void initLockManager() throws Exception {
this.lockManager = injector.getInstance(LockManager.class);
- if (lockManager != null) {
++ if (this.lockManager != null) {
+ logger.info("LockManager injection successful");
+ } else {
+ logger.error("LockManager injection unsuccessful");
+ }
}
-
/*
* (non-Javadoc)
*
[14/50] usergrid git commit: Add job id to log
Posted by sn...@apache.org.
Add job id to log
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/29d7660e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/29d7660e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/29d7660e
Branch: refs/heads/master
Commit: 29d7660eebdf471fe63c78c10b3c725bbbf40109
Parents: e0a9952
Author: Mike Dunker <md...@apigee.com>
Authored: Fri Apr 8 10:58:20 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Fri Apr 8 10:58:20 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/29d7660e/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
index 9b1fb05..83cac06 100644
--- a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
+++ b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
@@ -77,8 +77,8 @@ public abstract class OnlyOnceJob implements Job {
@Override
public void execute( JobExecution execution ) throws Exception {
- logger.info("Executing one-time job, LockManager is {}", lockManager == null ? "null" : "not null");
String lockId = execution.getJobId().toString();
+ logger.info("Executing one-time job {}, LockManager is {}", lockId, lockManager == null ? "null" : "not null");
Lock lock = lockManager.createLock( emf.getManagementAppId(), String.format( "/jobs/%s", lockId ) );
[28/50] usergrid git commit: Concurreny / threading changes. Remove
Quorum reads that aren't necessarily consistency related problems.
Posted by sn...@apache.org.
Concurreny / threading changes. Remove Quorum reads that aren't necessarily consistency related problems.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e935067f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e935067f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e935067f
Branch: refs/heads/master
Commit: e935067fa3b46368410a3f16b328d1dfc857eb7b
Parents: 8c0338c
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Apr 13 18:53:09 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Apr 13 18:53:09 2016 +0200
----------------------------------------------------------------------
tests/integration/test/notifications/notifications.js | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e935067f/tests/integration/test/notifications/notifications.js
----------------------------------------------------------------------
diff --git a/tests/integration/test/notifications/notifications.js b/tests/integration/test/notifications/notifications.js
index 644510f..7a255e5 100644
--- a/tests/integration/test/notifications/notifications.js
+++ b/tests/integration/test/notifications/notifications.js
@@ -278,7 +278,6 @@ module.exports = {
function (err, notification) {
should(err).be.null;
notification.should.not.be.null;
- notification.expectedCount.should.be.equal(1);
setTimeout(function() {
// wait a second before proceeding
@@ -306,7 +305,6 @@ module.exports = {
function (err, notification) {
should(err).be.null;
notification.should.not.be.null;
- notification.expectedCount.should.be.equal(1);
setTimeout(function() {
// wait a second before proceeding
@@ -334,9 +332,6 @@ module.exports = {
should(err).be.null;
notification.should.not.be.null;
- // we set up 2 groups of the same 5 users. if duplicate filtering is working,
- // we'll only have 5 expected
- notification.expectedCount.should.be.equal(5);
setTimeout(function() {
// wait a second before proceeding
[25/50] usergrid git commit: Merge branch 'release-2.1.1' of
https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
# By Michael Russo
# Via Michael Russo
* 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid:
Fix NoSuchElementException when no devices are found for the targeted push notification.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/17e9b36e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/17e9b36e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/17e9b36e
Branch: refs/heads/master
Commit: 17e9b36e55c4094715e87935d122fac5d76389a7
Parents: 19892a9 7c24d43
Author: George Reyes <gr...@apache.org>
Authored: Tue Apr 12 20:35:57 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Tue Apr 12 20:35:57 2016 -0700
----------------------------------------------------------------------
.../services/notifications/impl/ApplicationQueueManagerImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[40/50] usergrid git commit: Fix default value for saveReceipts
property on Notifications.
Posted by sn...@apache.org.
Fix default value for saveReceipts property on Notifications.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/22a8d059
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/22a8d059
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/22a8d059
Branch: refs/heads/master
Commit: 22a8d0595b49b2934ee208cd3fd60e272cd9601e
Parents: 2e344cb
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Apr 15 01:28:48 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Apr 15 01:28:48 2016 +0200
----------------------------------------------------------------------
.../usergrid/services/notifications/NotificationsService.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/22a8d059/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
index 824089a..05c1cd7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
@@ -139,7 +139,8 @@ public class NotificationsService extends AbstractCollectionService {
try {
validate(null, context.getPayload());
Notification.PathTokens pathTokens = getPathTokens(context.getRequest().getOriginalParameters());
- context.getProperties().put("saveReceipts", true); // default saving of receipts
+ // default saving of receipts
+ context.getProperties().put("saveReceipts", context.getProperties().getOrDefault("saveReceipts", true));
context.getProperties().put("processingFinished", 0L); // defaulting processing finished to 0
context.getProperties().put("state", Notification.State.CREATED);
context.getProperties().put("pathQuery", pathTokens);
[06/50] usergrid git commit: Revert "Make limit for versions to
search for configurable."
Posted by sn...@apache.org.
Revert "Make limit for versions to search for configurable."
This reverts commit 8c186a91ae44fe104ddfd16338cfa7f5bb36bf74.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/90217b52
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/90217b52
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/90217b52
Branch: refs/heads/master
Commit: 90217b527f23c12c9314b90661d19813c2fbe0dd
Parents: a976cff
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 5 13:47:34 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 5 13:47:34 2016 -0700
----------------------------------------------------------------------
.../corepersistence/asyncevents/EventBuilderImpl.java | 4 ++--
.../usergrid/corepersistence/index/IndexService.java | 5 +++--
.../corepersistence/index/IndexServiceImpl.java | 13 ++++---------
.../apache/usergrid/persistence/index/EntityIndex.java | 4 ++--
.../persistence/index/impl/EsEntityIndexImpl.java | 5 +++--
.../persistence/index/impl/EntityIndexTest.java | 4 ++--
6 files changed, 16 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/90217b52/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index c43b3d5..9851936 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -128,7 +128,7 @@ public class EventBuilderImpl implements EventBuilder {
if(mostRecentlyMarked != null){
deIndexObservable =
- indexService.deleteEntityIndexes( applicationScope, entityId, mostRecentlyMarked.getVersion(), false );
+ indexService.deleteEntityIndexes( applicationScope, entityId, mostRecentlyMarked.getVersion() );
ecmDeleteObservable =
ecm.getVersions( entityId )
@@ -194,7 +194,7 @@ public class EventBuilderImpl implements EventBuilder {
UUID latestVersion = latestVersions.getMaxVersion(entityId).getVersion();
deIndexObservable =
- indexService.deleteEntityIndexes( applicationScope, entityId, latestVersion, true );
+ indexService.deleteEntityIndexes( applicationScope, entityId, latestVersion);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/90217b52/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
index b48062a..54eb464 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -24,11 +24,13 @@ import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.index.IndexEdge;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
+import rx.observables.ConnectableObservable;
/**
@@ -76,11 +78,10 @@ public interface IndexService {
*
* @param applicationScope
* @param entityId
- * @param forEntityUpdates
* @return
*/
Observable<IndexOperationMessage> deleteEntityIndexes(final ApplicationScope applicationScope, final Id entityId,
- final UUID markedVersion, boolean forEntityUpdates);
+ final UUID markedVersion);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/90217b52/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 38e7122..6c07212 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -52,6 +52,7 @@ import rx.Observable;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createSearchEdgeFromSource;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromTarget;
+import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
/**
@@ -199,19 +200,13 @@ public class IndexServiceImpl implements IndexService {
//This should look up the entityId and delete any documents with a timestamp that comes before
//The edges that are connected will be compacted away from the graph.
@Override
- public Observable<IndexOperationMessage> deleteEntityIndexes(final ApplicationScope applicationScope,
- final Id entityId, final UUID markedVersion, boolean forEntityUpdates) {
+ public Observable<IndexOperationMessage> deleteEntityIndexes( final ApplicationScope applicationScope,
+ final Id entityId, final UUID markedVersion ) {
//bootstrap the lower modules from their caches
final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
- CandidateResults crs;
- if(forEntityUpdates){
- crs = ei.getAllEntityVersionsBeforeMarkedVersion( entityId, markedVersion, indexFig.getOldVersionQueryLimit() );
-
- }else{
- crs = ei.getAllEntityVersionsBeforeMarkedVersion( entityId, markedVersion, indexFig.getVersionQueryLimit() );
- }
+ CandidateResults crs = ei.getAllEntityVersionsBeforeMarkedVersion( entityId, markedVersion );
//If we get no search results, its possible that something was already deleted or
//that it wasn't indexed yet. In either case we can't delete anything and return an empty observable..
http://git-wip-us.apache.org/repos/asf/usergrid/blob/90217b52/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 4f7ee72..81f900c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.persistence.index;
+import com.google.common.base.Optional;
import org.apache.usergrid.persistence.core.CPManager;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -116,10 +117,9 @@ public interface EntityIndex extends CPManager {
*
* @param entityId The entityId to match when searching
* @param markedVersion The version that has been marked for deletion. All version before this one must be deleted.
- * @param limitToDelete
* @return
*/
- CandidateResults getAllEntityVersionsBeforeMarkedVersion(final Id entityId, final UUID markedVersion, int limitToDelete);
+ CandidateResults getAllEntityVersionsBeforeMarkedVersion(final Id entityId, final UUID markedVersion);
/**
* delete all application records
http://git-wip-us.apache.org/repos/asf/usergrid/blob/90217b52/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 98c0738..f754cc2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -56,6 +56,7 @@ import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
@@ -532,7 +533,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
@Override
- public CandidateResults getAllEntityVersionsBeforeMarkedVersion(final Id entityId, final UUID markedVersion, int limitToDelete) {
+ public CandidateResults getAllEntityVersionsBeforeMarkedVersion( final Id entityId, final UUID markedVersion ) {
Preconditions.checkNotNull( entityId, "entityId cannot be null" );
Preconditions.checkNotNull(markedVersion, "markedVersion cannot be null");
@@ -544,7 +545,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
final long markedTimestamp = markedVersion.timestamp();
// never let the limit be less than 2 as there are potential indefinite paging issues
- final int searchLimit = Math.max(2, limitToDelete);
+ final int searchLimit = Math.max(2, indexFig.getOldVersionQueryLimit());
// this query will find the document for the entity itself
final QueryBuilder entityQuery = QueryBuilders
http://git-wip-us.apache.org/repos/asf/usergrid/blob/90217b52/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 0896229..3978956 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -387,7 +387,7 @@ public class EntityIndexTest extends BaseIT {
* Tests that we aggregate results only before the halfway version point.
*/
@Test
- public void testScrollingDeindex() {
+ public void testScollingDeindex() {
int numberOfEntities = 1000;
int versionToSearchFor = numberOfEntities / 2;
@@ -419,7 +419,7 @@ public class EntityIndexTest extends BaseIT {
CandidateResults candidateResults = entityIndex
.getAllEntityVersionsBeforeMarkedVersion( entity[versionToSearchFor].getId(),
- entity[versionToSearchFor].getVersion(), numberOfEntities );
+ entity[versionToSearchFor].getVersion() );
assertEquals( 501, candidateResults.size() );
}
[22/50] usergrid git commit: Merge commit 'refs/pull/500/head' of
github.com:apache/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge commit 'refs/pull/500/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6b9e5efe
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6b9e5efe
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6b9e5efe
Branch: refs/heads/master
Commit: 6b9e5efe05a57cca90b4daff35917528c5417646
Parents: 5a508c6 44bcbc4
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 12 00:26:22 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 12 00:26:22 2016 +0200
----------------------------------------------------------------------
tests/performance/runLoadNestedEntities.sh | 98 ++++++++++++++++++++
.../datagenerators/EntityDataGenerator.scala | 40 ++++++++
.../org/apache/usergrid/enums/EntityType.scala | 3 +-
3 files changed, 140 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[09/50] usergrid git commit: USERGRID-1273: LockManager fix for
OnlyOnceJob.
Posted by sn...@apache.org.
USERGRID-1273: LockManager fix for OnlyOnceJob.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4326b105
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4326b105
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4326b105
Branch: refs/heads/master
Commit: 4326b105e6039b608969578e927e634e3f2eb620
Parents: 9f45130
Author: Mike Dunker <md...@apigee.com>
Authored: Wed Apr 6 20:12:58 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Wed Apr 6 20:12:58 2016 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/batch/job/OnlyOnceJob.java | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4326b105/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
index f857d05..82e856d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
+++ b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
@@ -19,7 +19,7 @@ package org.apache.usergrid.batch.job;
import java.util.concurrent.TimeUnit;
-import org.elasticsearch.common.inject.Inject;
+import com.google.inject.Injector;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.apache.usergrid.batch.Job;
@@ -28,6 +28,8 @@ import org.apache.usergrid.locking.Lock;
import org.apache.usergrid.locking.LockManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
+import javax.annotation.PostConstruct;
+
/**
* Simple abstract job class that performs additional locking to ensure that the job is only executing once. This can be
@@ -38,12 +40,14 @@ import org.apache.usergrid.persistence.EntityManagerFactory;
@Component("OnlyOnceJob")
public abstract class OnlyOnceJob implements Job {
- @Inject
private LockManager lockManager;
@Autowired
private EntityManagerFactory emf;
+ @Autowired
+ private Injector injector;
+
/**
*
@@ -51,6 +55,11 @@ public abstract class OnlyOnceJob implements Job {
public OnlyOnceJob() {
}
+ @PostConstruct
+ public void initLockManager() throws Exception {
+ this.lockManager = injector.getInstance(LockManager.class);
+ }
+
/*
* (non-Javadoc)
[34/50] usergrid git commit: Disable indexing per Entity type for
USERGRID-1278. https://issues.apache.org/jira/browse/USERGRID-1278
Posted by sn...@apache.org.
Disable indexing per Entity type for USERGRID-1278. https://issues.apache.org/jira/browse/USERGRID-1278
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fbced588
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fbced588
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fbced588
Branch: refs/heads/master
Commit: fbced588bdf7de5f7192d20e2da6a8740df0218a
Parents: 2e576f3
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Apr 14 12:26:06 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Apr 14 12:26:06 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 86 ++++++++++++++------
.../collection/CollectionsResourceIT.java | 56 +++++++++++--
2 files changed, 112 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbced588/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 891c1fe..b2330f3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -601,14 +601,36 @@ public class CpEntityManager implements EntityManager {
handleWriteUniqueVerifyException( entity, wuve );
}
- // queue an event to update the new entity
- indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+ if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
+ // queue an event to update the new entity
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
- // queue up an event to clean-up older versions than this one from the index
- if(entityManagerFig.getDeindexOnUpdate()) {
- indexService.queueDeIndexOldVersion(applicationScope, entityId);
+ // queue up an event to clean-up older versions than this one from the index
+ if (entityManagerFig.getDeindexOnUpdate()) {
+ indexService.queueDeIndexOldVersion( applicationScope, entityId );
+ }
+ }
+ }
+
+ private boolean skipIndexingForType( String type ) {
+
+ boolean skipIndexing = false;
+
+ MapManager mm = getMapManagerForTypes();
+ IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm );
+ String collectionName = Schema.defaultCollectionName( type );
+ Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName );
+
+ if ( collectionIndexingSchema.isPresent()) {
+ Map jsonMapData = collectionIndexingSchema.get();
+ final ArrayList fields = (ArrayList) jsonMapData.get( "fields" );
+ if ( fields.size() == 1 && fields.get(0).equals("!")) {
+ skipIndexing = true;
+ }
}
+
+ return skipIndexing;
}
@@ -670,8 +692,9 @@ public class CpEntityManager implements EntityManager {
Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() );
- //Step 4 && 5
- indexService.queueEntityDelete( applicationScope, entityId );
+ if ( !skipIndexingForType( entityId.getType() ) ) {
+ indexService.queueEntityDelete( applicationScope, entityId );
+ }
//Step 6
//delete from our UUID index
@@ -714,7 +737,8 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public Results searchCollectionConsistent( EntityRef entityRef, String collectionName, Query query, int expectedResults) throws Exception {
+ public Results searchCollectionConsistent(
+ EntityRef entityRef, String collectionName, Query query, int expectedResults) throws Exception {
return getRelationManager( entityRef ).searchCollectionConsistent(collectionName, query, expectedResults);
}
@@ -754,8 +778,8 @@ public class CpEntityManager implements EntityManager {
public RelationManager getRelationManager( EntityRef entityRef ) {
Preconditions.checkNotNull(entityRef, "entityRef cannot be null");
- CpRelationManager relationManager =
- new CpRelationManager(managerCache, indexService, collectionService, connectionService, this, entityManagerFig, applicationId, entityRef );
+ CpRelationManager relationManager = new CpRelationManager( managerCache, indexService, collectionService,
+ connectionService, this, entityManagerFig, applicationId, entityRef );
return relationManager;
}
@@ -830,7 +854,8 @@ public class CpEntityManager implements EntityManager {
Timer.Context repairedEntityGet = entGetRepairedEntityTimer.time();
- //TODO: can't we just sub in the getEntityRepair method here so for every read of a uniqueEntityField we can verify it is correct?
+ // TODO: can't we just sub in the getEntityRepair method here
+ // so for every read of a uniqueEntityField we can verify it is correct?
StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString());
@@ -916,8 +941,8 @@ public class CpEntityManager implements EntityManager {
// add a warn statement so we can see if we have data migration issues.
// TODO When we get an event system, trigger a repair if this is detected
if ( results.size() > 1 ) {
- logger.warn( "More than 1 entity with Owner id '{}' of type '{}' and alias '{}' exists. This is a duplicate alias, and needs audited",
- ownerRef, collectionType, aliasValue );
+ logger.warn( "More than 1 entity with Owner id '{}' of type '{}' and alias '{}' exists. " +
+ "This is a duplicate alias, and needs audited", ownerRef, collectionType, aliasValue );
}
return results.get(aliasValue);
@@ -1139,7 +1164,9 @@ public class CpEntityManager implements EntityManager {
//Adding graphite metrics
- indexService.queueEntityIndexUpdate(applicationScope, cpEntity, 0);
+ if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+ }
}
@@ -1785,12 +1812,17 @@ public class CpEntityManager implements EntityManager {
//do a check to see if you have a * field. If you do have a * field then ignore all other fields
//and only accept the * field.
- if(fieldProperties.contains( "*" )){
- ArrayList<String> wildCardArrayList = new ArrayList<>( );
+ if ( fieldProperties.contains( "*" )) {
+ ArrayList<String> wildCardArrayList = new ArrayList<>();
wildCardArrayList.add( "*" );
- schemaMap.put( "fields",wildCardArrayList );
- }
- else {
+ schemaMap.put( "fields", wildCardArrayList );
+
+ } else if ( fieldProperties.contains( "!" )) {
+ ArrayList<String> wildCardArrayList = new ArrayList<>();
+ wildCardArrayList.add( "!" );
+ schemaMap.put( "fields", wildCardArrayList );
+
+ } else {
schemaMap.putAll( properties );
}
@@ -1815,7 +1847,7 @@ public class CpEntityManager implements EntityManager {
public Object getCollectionSchema( String collectionName ){
MapManager mm = getMapManagerForTypes();
- IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm ); //managerCache.getIndexSchema( mm );
+ IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm );
Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName );
@@ -2388,7 +2420,8 @@ public class CpEntityManager implements EntityManager {
if ( !skipAggregateCounters ) {
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> m = createMutator( cass.getApplicationKeyspace( applicationId ), be );
- counterUtils.batchIncrementAggregateCounters( m, applicationId, userId, groupId, null, category, counters, timestamp );
+ counterUtils.batchIncrementAggregateCounters(
+ m, applicationId, userId, groupId, null, category, counters, timestamp );
//Adding graphite metrics
Timer.Context timeIncrementCounters =entIncrementAggregateCountersTimer.time();
@@ -2987,8 +3020,9 @@ public class CpEntityManager implements EntityManager {
final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId() );
- return graphManager.getEdgeTypesFromSource( searchByEdgeType ).map( edgeName -> getConnectionNameFromEdgeName( edgeName ) ).collect(
- () -> new HashSet<String>(), ( r, s ) -> r.add( s ) ).toBlocking().last();
+ return graphManager.getEdgeTypesFromSource(
+ searchByEdgeType ).map( edgeName -> getConnectionNameFromEdgeName( edgeName ) )
+ .collect( () -> new HashSet<String>(), ( r, s ) -> r.add( s ) ).toBlocking().last();
}
@@ -3001,7 +3035,8 @@ public class CpEntityManager implements EntityManager {
final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId() );
return graphManager.getEdgeTypesToTarget(searchByEdgeType).map(
- edgeName -> getConnectionNameFromEdgeName( edgeName ) ).collect( () -> new HashSet<String>( ), ( r, s ) -> r.add(s) ).toBlocking().last();
+ edgeName -> getConnectionNameFromEdgeName( edgeName ) )
+ .collect( () -> new HashSet<String>( ), ( r, s ) -> r.add(s) ).toBlocking().last();
}
@@ -3030,7 +3065,8 @@ public class CpEntityManager implements EntityManager {
try {
for (int i = 0; i < 20; i++) {
if (searchCollection(
- new SimpleEntityRef(org.apache.usergrid.persistence.entities.Application.ENTITY_TYPE, getApplicationId()),
+ new SimpleEntityRef(
+ org.apache.usergrid.persistence.entities.Application.ENTITY_TYPE, getApplicationId()),
InflectionUtils.pluralize("refresh"),
Query.fromQL("select * where uuid='" + refreshEntity.getUuid() + "'")
).size() > 0
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbced588/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index d052564..b240629 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.rest.applications.collection;
import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.entities.Application;
@@ -42,11 +43,7 @@ import javax.ws.rs.client.Invocation;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.ArrayList;
+import java.util.*;
import org.apache.commons.lang.NullArgumentException;
@@ -943,4 +940,53 @@ public class CollectionsResourceIT extends AbstractRestIT {
Assert.assertEquals( "Should not be more than one name property", -1, secondFred );
*/
}
+
+
+ /**
+ * Test that when schema is "!" entity gets saved but does not get indexed
+ */
+ @Test
+ public void postCollectionSchemaWithWildcardIndexNone() throws Exception {
+
+ // creating schema with no index wildcard and other fields that should be ignored
+ ArrayList<String> indexingArray = new ArrayList<>( );
+ indexingArray.add( "!" );
+ indexingArray.add( "one" );
+ indexingArray.add( "two" );
+ Entity payload = new Entity();
+ payload.put( "fields", indexingArray);
+
+ String randomizer = RandomStringUtils.randomAlphanumeric(10);
+ String collectionName = "col_" + randomizer;
+ app().collection( collectionName ).collection( "_indexes" ).post( payload );
+ refreshIndex();
+
+ // was the no-index wildcard saved and others ignored?
+ Collection collection = app().collection( collectionName ).collection( "_index" ).get();
+ LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData();
+ ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" );
+ assertTrue( schema.contains( "!" ) );
+ assertFalse( schema.contains( "one" ) );
+ assertFalse( schema.contains( "two" ) );
+
+ // post an entity with a name and a color
+ String entityName = "name_" + randomizer;
+ Entity postedEntity = this.app().collection( collectionName )
+ .post( new Entity().withProp("name", entityName).withProp( "color", "magenta" ) );
+
+ // should be able to get entity by ID
+ Entity getByIdEntity = app().collection( collectionName ).entity( postedEntity.getUuid() ).get();
+ assertNotNull( getByIdEntity );
+ assertEquals( postedEntity.getUuid(), getByIdEntity.getUuid() );
+
+ // should be able to get entity by name
+ Entity getByNameEntity = app().collection( collectionName ).entity( entityName ).get();
+ assertNotNull( getByNameEntity );
+ assertEquals( postedEntity.getUuid(), getByNameEntity.getUuid() );
+
+ // should NOT be able to get entity by query
+ Iterator<Entity> getByQuery = app().collection( collectionName )
+ .get( new QueryParameters().setQuery( "select * where color='magenta'" ) ).iterator();
+ assertFalse( getByQuery.hasNext() );
+ }
}
[16/50] usergrid git commit: Don't cache Optionals or nulls in app id
cache.
Posted by sn...@apache.org.
Don't cache Optionals or nulls in app id cache.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/177e0637
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/177e0637
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/177e0637
Branch: refs/heads/master
Commit: 177e06375351dbcff7f14e158814b47c53413d56
Parents: 74de4bc
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Apr 8 18:51:14 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Apr 8 18:51:14 2016 -0400
----------------------------------------------------------------------
.../corepersistence/ApplicationIdCache.java | 6 +-
.../corepersistence/ApplicationIdCacheImpl.java | 64 ++++++++------------
.../corepersistence/CpEntityManagerFactory.java | 9 ++-
.../persistence/EntityManagerFactory.java | 4 +-
.../index/AsyncEventServiceImplTest.java | 23 +++----
.../cassandra/EntityManagerFactoryImplIT.java | 8 +--
.../apache/usergrid/mongo/BasicMongoTest.java | 8 +--
.../apache/usergrid/mongo/MongoQueryTest.java | 18 +++---
.../organizations/OrganizationResource.java | 8 +--
.../cassandra/ManagementServiceImpl.java | 8 +--
.../migration/AppInfoMigrationPluginTest.java | 8 +--
11 files changed, 70 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCache.java
index a7d1c4e..83c2144 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCache.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCache.java
@@ -35,17 +35,17 @@ public interface ApplicationIdCache {
/**
* Evict the application by name
*/
- public Optional<UUID> getApplicationId(final String applicationName);
+ UUID getApplicationId( final String applicationName );
/**
* Evict the app id by the name
*/
- public void evictAppId(final String applicationName);
+ void evictAppId( final String applicationName );
/**
* Evict all caches
*/
- public void evictAll();
+ void evictAll();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCacheImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCacheImpl.java
index 95171a8..7745359 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCacheImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/ApplicationIdCacheImpl.java
@@ -16,74 +16,64 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.usergrid.corepersistence;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import com.google.common.base.Optional;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-
/**
- * Implements the org app cache for faster runtime lookups. These values are immutable, so this LRU cache can stay
- * full for the duration of the execution
+ * Implements the org app cache for faster runtime lookups.
+ * These values are immutable, so this LRU cache can stay full for the duration of the execution.
*/
public class ApplicationIdCacheImpl implements ApplicationIdCache {
private static final Logger logger = LoggerFactory.getLogger(ApplicationIdCacheImpl.class);
+ // cache the pointer to our root entity manager for reference
+ private final LoadingCache<String, UUID> appCache;
- /**
- * Cache the pointer to our root entity manager for reference
- */
-
- private final LoadingCache<String, Optional<UUID>> appCache;
private final EntityManager managementEnityManager;
+
private final ManagerCache managerCache;
- public ApplicationIdCacheImpl(final EntityManager managementEnityManager, ManagerCache managerCache, ApplicationIdCacheFig fig) {
+ public ApplicationIdCacheImpl(
+ final EntityManager managementEnityManager, ManagerCache managerCache, ApplicationIdCacheFig fig) {
+
this.managementEnityManager = managementEnityManager;
this.managerCache = managerCache;
+
appCache = CacheBuilder.newBuilder()
.maximumSize(fig.getCacheSize())
.expireAfterWrite(fig.getCacheTimeout(), TimeUnit.MILLISECONDS)
- .build(new CacheLoader<String, Optional<UUID>>() {
+ .build(new CacheLoader<String, UUID>() {
@Override
- public Optional<UUID> load(final String key) throws Exception {
- return Optional.fromNullable(fetchApplicationId(key));
+ public UUID load(final String key) throws Exception {
+ return fetchApplicationId(key);
}
});
}
@Override
- public Optional<UUID> getApplicationId( final String applicationName ) {
+ public UUID getApplicationId( final String applicationName ) {
try {
- Optional<UUID> optionalUuid = appCache.get( applicationName.toLowerCase() );
- if(!optionalUuid.isPresent()){
- appCache.invalidate(applicationName.toLowerCase());
- return Optional.absent();
- }
- return optionalUuid;
+ return appCache.get( applicationName.toLowerCase() );
} catch (Exception e) {
if (logger.isDebugEnabled()) {
logger.debug("Returning for key {} value null", applicationName);
@@ -112,25 +102,21 @@ public class ApplicationIdCacheImpl implements ApplicationIdCache {
logger.error("Error looking up management app", e);
}
- try {
+ // look up application_info ID for application using unique "name" field
+ final Observable<Id> idObs = ecm.getIdField(
+ CpNamingUtils.APPLICATION_INFO, new StringField(Schema.PROPERTY_NAME, applicationName));
+ Id id = idObs.toBlocking().lastOrDefault(null);
- // look up application_info ID for application using unique "name" field
- final Observable<Id> idObs = ecm.getIdField(
- CpNamingUtils.APPLICATION_INFO, new StringField(Schema.PROPERTY_NAME, applicationName));
+ if ( id != null ) {
+ value = id.getUuid();
- Id id = idObs.toBlocking().lastOrDefault(null);
- if(id != null) {
- value = id.getUuid();
- }else{
- if(logger.isDebugEnabled()) {
- logger.debug("Could not load value for key {} ", applicationName);
- }
+ } else {
+ if (logger.isDebugEnabled()) {
+ logger.debug( "Could not load value for key {} ", applicationName );
}
- return value;
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Unable to retrieve application id", e );
}
+
+ return value;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index a2054d9..a88bf7b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -31,7 +31,6 @@ import org.springframework.context.ApplicationContextAware;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
-import org.apache.usergrid.corepersistence.index.IndexSchemaCache;
import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
import org.apache.usergrid.corepersistence.index.ReIndexService;
@@ -220,9 +219,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
String appName = buildAppName( orgName, name );
- final Optional<UUID> appId = applicationIdCache.getApplicationId( appName );
+ final UUID appId = applicationIdCache.getApplicationId( appName );
- if ( appId.isPresent()) {
+ if ( appId == null ) {
throw new ApplicationAlreadyExistsException( name );
}
@@ -260,7 +259,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// check for pre-existing application
- if ( lookupApplication( appName ).isPresent()) {
+ if ( lookupApplication( appName ) == null ) {
throw new ApplicationAlreadyExistsException( appName );
}
@@ -451,7 +450,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- public Optional<UUID> lookupApplication( String orgAppName ) throws Exception {
+ public UUID lookupApplication(String orgAppName ) throws Exception {
return applicationIdCache.getApplicationId(orgAppName);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index 96bd3b2..eedde17 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -20,8 +20,6 @@ package org.apache.usergrid.persistence;
import java.util.Map;
import java.util.UUID;
-import com.google.common.base.Optional;
-
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.index.EntityIndex;
@@ -121,7 +119,7 @@ public interface EntityManagerFactory {
*
* @throws Exception the exception
*/
- Optional<UUID> lookupApplication( String name ) throws Exception;
+ UUID lookupApplication(String name ) throws Exception;
/**
* Returns all the applications in the system.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncEventServiceImplTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncEventServiceImplTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncEventServiceImplTest.java
index c915464..4c0058b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncEventServiceImplTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncEventServiceImplTest.java
@@ -20,31 +20,24 @@
package org.apache.usergrid.corepersistence.index;
-import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.impl.IndexProducer;
-import org.apache.usergrid.persistence.queue.QueueFig;
-import org.junit.Rule;
-import org.junit.runner.RunWith;
-
+import com.google.inject.Inject;
+import net.jcip.annotations.NotThreadSafe;
import org.apache.usergrid.corepersistence.TestIndexModule;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventServiceImpl;
+import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.persistence.core.aws.NoAWSCredsRule;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.rx.RxTaskScheduler;
import org.apache.usergrid.persistence.core.test.UseModules;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.impl.EsRunner;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.queue.QueueFig;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
-
-import com.google.inject.Inject;
-
-import net.jcip.annotations.NotThreadSafe;
-
-import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import org.junit.Rule;
+import org.junit.runner.RunWith;
@RunWith( EsRunner.class )
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index abd55ea..d287d7e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -134,7 +134,7 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
Entity film2 = em.create( "film", properties2 );
for ( int j=0; j<maxRetries; j++ ) {
- if ( setup.getEmf().lookupApplication( orgName + "/" + appName ).isPresent()) {
+ if ( setup.getEmf().lookupApplication( orgName + "/" + appName ) != null ) {
break;
}
Thread.sleep( 500 );
@@ -171,7 +171,7 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
assertTrue("Deleted app must be found in in deleted apps collection", found);
// attempt to get entities in application's collections in various ways should all fail
- found = setup.getEmf().lookupApplication( orgName + "/" + appName ).isPresent() ;
+ found = setup.getEmf().lookupApplication( orgName + "/" + appName ) != null;
assertFalse("Lookup of deleted app must fail", found);
@@ -211,7 +211,7 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
assertTrue("Restored app not found in apps collection", found);
// TODO: this assertion should work!
- assertTrue(setup.getEmf().lookupApplication( orgName + "/" + appName ).isPresent());
+ assertTrue(setup.getEmf().lookupApplication( orgName + "/" + appName ) != null );
}
@@ -307,7 +307,7 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
UUID appId = setup.createApplication(orgName, appName);
- UUID lookedUpId = setup.getEmf().lookupApplication( orgAppName ).get();
+ UUID lookedUpId = setup.getEmf().lookupApplication( orgAppName );
Assert.assertEquals(appId, lookedUpId);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/BasicMongoTest.java
----------------------------------------------------------------------
diff --git a/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/BasicMongoTest.java b/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/BasicMongoTest.java
index 17e4449..bcd4d2b 100644
--- a/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/BasicMongoTest.java
+++ b/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/BasicMongoTest.java
@@ -110,7 +110,7 @@ public class BasicMongoTest extends AbstractMongoTest {
// check we can find it when using the native entity manager
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Entity entity = em.get( new SimpleEntityRef( (String)returnedObject.get("type"), id ));
@@ -228,7 +228,7 @@ public class BasicMongoTest extends AbstractMongoTest {
Thread.sleep( 5000 );
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Entity entity = em.get( new SimpleEntityRef( (String)returnedObject.get("type"), id ) );
@@ -290,7 +290,7 @@ public class BasicMongoTest extends AbstractMongoTest {
// check it has been deleted
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Entity entity = em.get( new SimpleEntityRef( (String)returnedObject.get("type"), id ) );
@@ -345,7 +345,7 @@ public class BasicMongoTest extends AbstractMongoTest {
assertFalse( cursor.hasNext() );
// check it has been deleted
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Results results =
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/MongoQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/MongoQueryTest.java b/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/MongoQueryTest.java
index 6bd1df9..98bc7ba 100644
--- a/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/MongoQueryTest.java
+++ b/stack/mongo-emulator/src/test/java/org/apache/usergrid/mongo/MongoQueryTest.java
@@ -45,7 +45,7 @@ public class MongoQueryTest extends AbstractMongoTest {
@Test
public void stringEqual() throws Exception {
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Map<String, Object> properties = new LinkedHashMap<String, Object>();
@@ -105,7 +105,7 @@ public class MongoQueryTest extends AbstractMongoTest {
@Test
public void greaterThan() throws Exception {
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Map<String, Object> properties = new LinkedHashMap<String, Object>();
@@ -156,7 +156,7 @@ public class MongoQueryTest extends AbstractMongoTest {
@Test
public void greaterThanEqual() throws Exception {
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Map<String, Object> properties = new LinkedHashMap<String, Object>();
@@ -211,7 +211,7 @@ public class MongoQueryTest extends AbstractMongoTest {
@Test
public void lessThan() throws Exception {
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Map<String, Object> properties = new LinkedHashMap<String, Object>();
@@ -262,7 +262,7 @@ public class MongoQueryTest extends AbstractMongoTest {
@Test
public void lessThanEqual() throws Exception {
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Map<String, Object> properties = new LinkedHashMap<String, Object>();
@@ -317,7 +317,7 @@ public class MongoQueryTest extends AbstractMongoTest {
@Test
public void in() throws Exception {
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Map<String, Object> properties = new LinkedHashMap<String, Object>();
@@ -372,7 +372,7 @@ public class MongoQueryTest extends AbstractMongoTest {
@Test
public void or() throws Exception {
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Map<String, Object> properties = new LinkedHashMap<String, Object>();
@@ -427,7 +427,7 @@ public class MongoQueryTest extends AbstractMongoTest {
@Test
public void and() throws Exception {
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Map<String, Object> properties = new LinkedHashMap<String, Object>();
@@ -476,7 +476,7 @@ public class MongoQueryTest extends AbstractMongoTest {
@Test
public void withFieldSelector() throws Exception {
- UUID appId = emf.lookupApplication( "test-organization/test-app" ).get();
+ UUID appId = emf.lookupApplication( "test-organization/test-app" );
EntityManager em = emf.getEntityManager( appId );
Map<String, Object> properties = new LinkedHashMap<String, Object>();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/rest/src/main/java/org/apache/usergrid/rest/organizations/OrganizationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/organizations/OrganizationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/organizations/OrganizationResource.java
index 43867e3..dea727f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/organizations/OrganizationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/organizations/OrganizationResource.java
@@ -138,9 +138,9 @@ public class OrganizationResource extends AbstractContextResource {
}
String orgAppName = PathingUtils.assembleAppName( organizationName, applicationName );
- Optional<UUID> optionalAppId = emf.lookupApplication( orgAppName );
+ UUID optionalAppId = emf.lookupApplication( orgAppName );
- if ( !optionalAppId.isPresent()) {
+ if ( optionalAppId == null ) {
// TODO: fix this hacky work-around for apparent Jersey issue
UUID applicationId = UUIDUtils.tryExtractUUID( applicationName );
@@ -148,11 +148,11 @@ public class OrganizationResource extends AbstractContextResource {
if ( applicationId == null ) {
throw new OrganizationApplicationNotFoundException( orgAppName, uriInfo, properties, management );
}else{
- optionalAppId = Optional.fromNullable(applicationId);
+ optionalAppId = applicationId;
}
}
- return appResourceFor( optionalAppId.get() );
+ return appResourceFor( optionalAppId );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 7b55c84..a586c26 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -1796,7 +1796,7 @@ public class ManagementServiceImpl implements ManagementService {
throw new EntityNotFoundException("Deleted application ID " + applicationId + " not found");
}
- if ( emf.lookupApplication( app.getName() ).isPresent()) {
+ if ( emf.lookupApplication( app.getName() ) != null ) {
throw new ConflictException("Cannot restore application, one with that name already exists.");
}
@@ -1990,11 +1990,11 @@ public class ManagementServiceImpl implements ManagementService {
if ( applicationName == null ) {
return null;
}
- Optional<UUID> applicationId = emf.lookupApplication(applicationName);
- if ( !applicationId.isPresent() ) {
+ UUID applicationId = emf.lookupApplication(applicationName);
+ if ( applicationId == null ) {
return null;
}
- return new ApplicationInfo( applicationId.get(), applicationName.toLowerCase() );
+ return new ApplicationInfo( applicationId, applicationName.toLowerCase() );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/177e0637/stack/services/src/test/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPluginTest.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPluginTest.java b/stack/services/src/test/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPluginTest.java
index 8fe50de..b697f12 100644
--- a/stack/services/src/test/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPluginTest.java
+++ b/stack/services/src/test/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPluginTest.java
@@ -202,7 +202,7 @@ public class AppInfoMigrationPluginTest {
for ( Entity applicationInfo : deletedApps ) {
String appName = applicationInfo.getName();
- boolean isPresent = setup.getEmf().lookupApplication( appName ).isPresent();
+ boolean isPresent = setup.getEmf().lookupApplication( appName ) != null;
// missing application_info does not completely break applications, but we...
assertFalse("Should not be able to lookup deleted application by name" + appName, isPresent);
@@ -219,11 +219,11 @@ public class AppInfoMigrationPluginTest {
String appName = orgName + "/application" + i;
- Optional<UUID> uuid = setup.getEmf().lookupApplication(appName);
+ UUID uuid = setup.getEmf().lookupApplication(appName);
- assertTrue ("Should be able to get application", uuid.isPresent() );
+ assertTrue ("Should be able to get application", uuid != null );
- EntityManager em = setup.getEmf().getEntityManager( uuid.get() );
+ EntityManager em = setup.getEmf().getEntityManager( uuid );
Application app = em.getApplication();
assertEquals( appName, app.getName() );
[38/50] usergrid git commit: Add receipts property to the
notification entity so it gets persisted.
Posted by sn...@apache.org.
Add receipts property to the notification entity so it gets persisted.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8dce5ed8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8dce5ed8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8dce5ed8
Branch: refs/heads/master
Commit: 8dce5ed8c26a27cd991375803327407f12efc3ca
Parents: 226cb62
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Apr 15 01:00:58 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Apr 15 01:00:58 2016 +0200
----------------------------------------------------------------------
.../persistence/entities/Notification.java | 13 +++++++++
.../notifications/NotificationsService.java | 2 ++
.../impl/ApplicationQueueManagerImpl.java | 30 ++++++--------------
3 files changed, 24 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8dce5ed8/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
index 1350593..6a6e3fa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
@@ -93,6 +93,10 @@ public class Notification extends TypedEntity {
@EntityProperty
protected String errorMessage;
+ /** Flag to disable the creation, saving, connecting of receipt entities for a notification. */
+ @EntityProperty
+ protected boolean saveReceipts;
+
@EntityCollection(type = "receipt")
protected List<UUID> receipts;
@@ -178,6 +182,15 @@ public class Notification extends TypedEntity {
}
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ public boolean getSaveReceipts() {
+ return saveReceipts;
+ }
+
+ public void setSaveReceipts(boolean saveReceipts) {
+ this.saveReceipts = saveReceipts;
+ }
+
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public boolean getDebug() {
return debug;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8dce5ed8/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
index bbdec7a..824089a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
@@ -139,6 +139,8 @@ public class NotificationsService extends AbstractCollectionService {
try {
validate(null, context.getPayload());
Notification.PathTokens pathTokens = getPathTokens(context.getRequest().getOriginalParameters());
+ context.getProperties().put("saveReceipts", true); // default saving of receipts
+ context.getProperties().put("processingFinished", 0L); // defaulting processing finished to 0
context.getProperties().put("state", Notification.State.CREATED);
context.getProperties().put("pathQuery", pathTokens);
context.setOwner(sm.getApplication());
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8dce5ed8/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 8b908ee..faa9a02 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -329,33 +329,21 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
String notifierName = message.getNotifierKey().toLowerCase();
ProviderAdapter providerAdapter = notifierMap.get(notifierName.toLowerCase());
Object payload = translatedPayloads.get(notifierName);
- Map dynamicNotificationProperties =notification.getDynamicProperties();
- //do additional error checking to make sure other values will work.
- Boolean receiptsField = true;
-
- Object typelessReceiptsField = dynamicNotificationProperties.getOrDefault( "receipts",true );
- if(typelessReceiptsField instanceof Boolean) {
- receiptsField = ( boolean ) typelessReceiptsField;
- }
- else if(typelessReceiptsField instanceof String){
- String booleanString = ( String ) typelessReceiptsField;
- if(booleanString.toLowerCase().equals( "false" )){
- receiptsField=false;
- }
- }
TaskTracker tracker = null;
- if(receiptsField==false){
-// Receipt receipt =
-// new Receipt( notification.getUuid(), message.getNotifierId(), payload, deviceUUID );
+
+ if(notification.getSaveReceipts()){
+
+ final Receipt receipt =
+ new Receipt( notification.getUuid(), message.getNotifierId(), payload, deviceUUID );
tracker =
- new TaskTracker( providerAdapter.getNotifier(), taskManager, null, deviceUUID );
+ new TaskTracker( providerAdapter.getNotifier(), taskManager, receipt, deviceUUID );
+
}
else {
- Receipt receipt =
- new Receipt( notification.getUuid(), message.getNotifierId(), payload, deviceUUID );
+
tracker =
- new TaskTracker( providerAdapter.getNotifier(), taskManager, receipt, deviceUUID );
+ new TaskTracker( providerAdapter.getNotifier(), taskManager, null, deviceUUID );
}
if (!isOkToSend(notification)) {
tracker.failed(0, "Notification is duplicate/expired/cancelled.");
[26/50] usergrid git commit: Push notification queuing to be async.
Posted by sn...@apache.org.
Push notification queuing to be async.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/11274801
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/11274801
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/11274801
Branch: refs/heads/master
Commit: 11274801059723b4a2264e01f45141080a47511a
Parents: 17e9b36
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Apr 13 15:51:08 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Apr 13 15:51:08 2016 +0200
----------------------------------------------------------------------
.../persistence/entities/Notification.java | 10 -----
.../impl/ApplicationQueueManagerImpl.java | 45 +++++++-------------
2 files changed, 16 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11274801/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
index 5c3ee89..dc7c989 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
@@ -41,11 +41,6 @@ public class Notification extends TypedEntity {
NORMAL, HIGH
}
-
- /** Total count of notifications sent based on the API path/query */
- @EntityProperty
- protected int expectedCount;
-
/** The pathQuery/query that Usergrid used to idenitfy the devices to send the notification to */
@EntityProperty
private PathTokens pathQuery;
@@ -107,11 +102,6 @@ public class Notification extends TypedEntity {
}
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
- public int getExpectedCount() { return expectedCount; }
-
- public void setExpectedCount(int expectedCount) { this.expectedCount = expectedCount; }
-
- @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public PathTokens getPathQuery(){
return pathQuery;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11274801/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index fb4d64c..f819e39 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -20,10 +20,7 @@ import com.codahale.metrics.Meter;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.entities.Device;
-import org.apache.usergrid.persistence.entities.Notification;
-import org.apache.usergrid.persistence.entities.Notifier;
-import org.apache.usergrid.persistence.entities.Receipt;
+import org.apache.usergrid.persistence.entities.*;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueMessage;
@@ -33,6 +30,7 @@ import org.slf4j.LoggerFactory;
import rx.Observable;
import rx.Subscriber;
import rx.functions.Func1;
+import rx.schedulers.Schedulers;
import java.io.IOException;
import java.util.*;
@@ -174,15 +172,18 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
//process up to 10 concurrently
Observable processMessagesObservable = Observable.create(new IteratorObservable<Entity>(iterator))
.flatMap(entity -> {
+
+ if(entity.getType().equals(Device.ENTITY_TYPE)){
+ return Observable.from(Collections.singletonList(entity));
+ }
+
+ // if it's not a device, drill down and get them
return Observable.from(getDevices(entity));
- }, 10)
+
+ }, 50)
.distinct(ref -> ref.getUuid())
.map(sendMessageFunction)
- .buffer(100)
- .doOnNext( applicationQueueMessages -> {
-
- applicationQueueMessages.forEach( message -> {
-
+ .doOnNext( message -> {
try {
if(message.isPresent()){
qm.sendMessage( message.get() );
@@ -201,13 +202,12 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}
- });
-
})
.doOnError(throwable -> logger.error("Failed while trying to send notification", throwable));
- processMessagesObservable.toBlocking().lastOrDefault(null); // let this run and block the async thread, messages are queued
+ //TODO verify error handling here
+ processMessagesObservable.subscribeOn(Schedulers.io()).subscribe(); // fire the queuing into the background
}
@@ -221,7 +221,6 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}
}
- notification.setExpectedCount(deviceCount.get());
notification.addProperties(properties);
em.update(notification);
@@ -491,14 +490,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
private boolean isOkToSend(Notification notification) {
- Map<String, Long> stats = notification.getStatistics();
- if (stats != null && notification.getExpectedCount() == (stats.get("sent") + stats.get("errors"))) {
- if (logger.isDebugEnabled()) {
- logger.debug("notification {} already processed. not sending.",
- notification.getUuid());
- }
- return false;
- }
+
if (notification.getCanceled() == Boolean.TRUE) {
if (logger.isDebugEnabled()) {
logger.debug("notification {} canceled. not sending.",
@@ -522,14 +514,9 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
final int LIMIT = Query.MID_LIMIT;
-
try {
- if ("device".equals(ref.getType())) {
-
- devices = Collections.singletonList(ref);
-
- } else if ("user".equals(ref.getType())) {
+ if (User.ENTITY_TYPE.equals(ref.getType())) {
UUID start = null;
boolean initial = true;
@@ -551,7 +538,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}
- } else if ("group".equals(ref.getType())) {
+ } else if (Group.ENTITY_TYPE.equals(ref.getType())) {
UUID start = null;
boolean initial = true;
[21/50] usergrid git commit: Merge commit 'refs/pull/499/head' of
github.com:apache/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge commit 'refs/pull/499/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5a508c6e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5a508c6e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5a508c6e
Branch: refs/heads/master
Commit: 5a508c6e63575efcf3f997a971b11b84f33b0dae
Parents: c1375bf 9486cdd
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 12 00:26:05 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 12 00:26:05 2016 +0200
----------------------------------------------------------------------
.../corepersistence/ApplicationIdCache.java | 6 +-
.../corepersistence/ApplicationIdCacheImpl.java | 64 ++++++++------------
.../corepersistence/CpEntityManagerFactory.java | 9 ++-
.../persistence/EntityManagerFactory.java | 4 +-
.../index/AsyncEventServiceImplTest.java | 23 +++----
.../cassandra/EntityManagerFactoryImplIT.java | 8 +--
.../apache/usergrid/mongo/BasicMongoTest.java | 8 +--
.../apache/usergrid/mongo/MongoQueryTest.java | 18 +++---
.../organizations/OrganizationResource.java | 8 +--
.../cassandra/ManagementServiceImpl.java | 8 +--
.../migration/AppInfoMigrationPluginTest.java | 8 +--
11 files changed, 70 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
[23/50] usergrid git commit: Fix NoSuchElementException when no
devices are found for the targeted push notification.
Posted by sn...@apache.org.
Fix NoSuchElementException when no devices are found for the targeted push notification.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7c24d43b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7c24d43b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7c24d43b
Branch: refs/heads/master
Commit: 7c24d43b5c8fc95d049ce02bc3bc7dbac69f6587
Parents: 6b9e5ef
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Apr 13 00:15:32 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Apr 13 00:15:32 2016 +0200
----------------------------------------------------------------------
.../services/notifications/impl/ApplicationQueueManagerImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7c24d43b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 6c28d2f..fb4d64c 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -207,7 +207,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
})
.doOnError(throwable -> logger.error("Failed while trying to send notification", throwable));
- processMessagesObservable.toBlocking().last(); // let this run and block the async thread, messages are queued
+ processMessagesObservable.toBlocking().lastOrDefault(null); // let this run and block the async thread, messages are queued
}
[42/50] usergrid git commit: Remove parallel processing of the
iterator.
Posted by sn...@apache.org.
Remove parallel processing of the iterator.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f064c499
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f064c499
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f064c499
Branch: refs/heads/master
Commit: f064c499c042289c5bc31cdda7b34199701ec795
Parents: 30266e1
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Apr 15 10:12:23 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Apr 15 10:12:23 2016 +0200
----------------------------------------------------------------------
.../services/notifications/impl/ApplicationQueueManagerImpl.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f064c499/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index faa9a02..2466164 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -168,8 +168,6 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
};
-
- //process up to 10 concurrently
Observable processMessagesObservable = Observable.create(new IteratorObservable<Entity>(iterator))
.flatMap(entity -> {
@@ -180,7 +178,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
// if it's not a device, drill down and get them
return Observable.from(getDevices(entity));
- }, 50)
+ })
.distinct(ref -> ref.getUuid())
.map(sendMessageFunction)
.doOnNext( message -> {
[37/50] usergrid git commit: Fix issues when groups not being
targeted in notifications when referencing by name.
Posted by sn...@apache.org.
Fix issues when groups not being targeted in notifications when referencing by name.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/226cb62b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/226cb62b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/226cb62b
Branch: refs/heads/master
Commit: 226cb62b5c6ebf76741054d8a93e09265af21079
Parents: 99e6d40
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Apr 15 00:48:01 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Apr 15 00:48:01 2016 +0200
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 7 ++++++
.../apache/usergrid/persistence/PathQuery.java | 24 ++++++++++++++++++++
2 files changed, 31 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/226cb62b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index b5a4107..9ecf466 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -945,6 +945,13 @@ public class CpRelationManager implements RelationManager {
query.setQl( newQuery );
}
+ // groups have a special unique identifier
+ else if ( query.getEntityType().equals( Group.ENTITY_TYPE ) ){
+
+ final String newQuery = "select * where path='" + query.getSingleNameOrEmailIdentifier() + "'";
+
+ query.setQl( newQuery );
+ }
// use the ident with the default alias. could be an email
else {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/226cb62b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
index bb336e1..55839a6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
@@ -17,10 +17,13 @@
package org.apache.usergrid.persistence;
+import java.util.Collections;
import java.util.Iterator;
import java.util.UUID;
import org.apache.usergrid.persistence.Query.Level;
+import org.apache.usergrid.persistence.index.query.Identifier;
+import org.apache.usergrid.utils.InflectionUtils;
public class PathQuery<E> {
@@ -85,6 +88,7 @@ public class PathQuery<E> {
public Iterator<E> iterator( EntityManager em ) {
try {
+
if ( uuid != null && type != null ) {
return new PagingResultsIterator( getHeadResults( em ), query.getResultsLevel() );
}
@@ -99,7 +103,20 @@ public class PathQuery<E> {
protected Results getHeadResults( EntityManager em ) throws Exception {
+
EntityRef ref = new SimpleEntityRef(type,uuid);
+
+ // if it's a single name identifier, just directly fetch that
+ if ( !query.getQl().isPresent() && query.getSingleNameOrEmailIdentifier() != null){
+
+ String name = query.getSingleNameOrEmailIdentifier();
+ String entityType = InflectionUtils.singularize(query.getCollection());
+
+ UUID entityId = em.getUniqueIdFromAlias( entityType, name );
+
+ return em.getEntities(Collections.singletonList(entityId), entityType);
+ }
+
return ( query.getCollection() != null ) ?
em.searchCollection( ref, query.getCollection(), query ) :
em.searchTargetEntities(ref, query);
@@ -107,6 +124,13 @@ public class PathQuery<E> {
protected Iterator refIterator( EntityManager em ) throws Exception {
+
+ if ( query.getQl() == null && query.getSingleNameOrEmailIdentifier() != null){
+
+ return new PagingResultsIterator( getHeadResults( em ), Level.REFS );
+
+ }
+
if ( type != null && uuid != null) {
return new PagingResultsIterator( getHeadResults( em ), Level.REFS );
}
[30/50] usergrid git commit: Merge branch 'release-2.1.1' of
https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge branch 'release-2.1.1' of https://git-wip-us.apache.org/repos/asf/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a4622441
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a4622441
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a4622441
Branch: refs/heads/master
Commit: a46224419cadfb71b3efd3f3005fce4212dcebcc
Parents: 794bbd4 e935067
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Apr 13 18:55:19 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Apr 13 18:55:19 2016 +0200
----------------------------------------------------------------------
----------------------------------------------------------------------
[32/50] usergrid git commit: Make the intial write path of indexing
require less I/O.
Posted by sn...@apache.org.
Make the intial write path of indexing require less I/O.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/10ff27cc
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/10ff27cc
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/10ff27cc
Branch: refs/heads/master
Commit: 10ff27cc86923a24c52e881f821870867dd95212
Parents: a462244
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Apr 13 22:42:12 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Apr 13 22:42:12 2016 +0200
----------------------------------------------------------------------
.../asyncevents/AsyncEventServiceImpl.java | 102 ++++++++++++++-----
.../asyncevents/model/AsyncEvent.java | 2 +
.../asyncevents/model/EdgeIndexEvent.java | 70 +++++++++++++
.../asyncevents/model/EntityIndexEvent.java | 54 ++++++++++
4 files changed, 200 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/10ff27cc/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 82ad5be..3b01292 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -310,20 +310,33 @@ public class AsyncEventServiceImpl implements AsyncEventService {
logger.debug("Processing event with type {}", event.getClass().getSimpleName());
}
- IndexOperationMessage indexOperationMessage = null;
try {
+ IndexOperationMessage single = new IndexOperationMessage();
+
+ // normal indexing event for an entity
+ if ( event instanceof EntityIndexEvent ){
+
+ single = handleEntityIndexUpdate( message );
+
+ }
+ // normal indexing event for an edge
+ else if ( event instanceof EdgeIndexEvent ){
+
+ single = handleEdgeIndex( message );
+
+ }
// deletes are 2-part, actual IO to delete data, then queue up a de-index
- if ( event instanceof EdgeDeleteEvent ) {
+ else if ( event instanceof EdgeDeleteEvent ) {
- handleEdgeDelete( message );
+ single = handleEdgeDelete( message );
}
// deletes are 2-part, actual IO to delete data, then queue up a de-index
else if ( event instanceof EntityDeleteEvent ) {
- handleEntityDelete( message );
+ single = handleEntityDelete( message );
}
- // application initialization has special logic, therefore a special event type
+ // initialization has special logic, therefore a special event type and no index operation message
else if ( event instanceof InitializeApplicationIndexEvent ) {
handleInitializeApplicationIndex(event, message);
@@ -331,11 +344,11 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// this is the main event that pulls the index doc from map persistence and hands to the index producer
else if (event instanceof ElasticsearchIndexEvent) {
- indexOperationMessage = handleIndexOperation((ElasticsearchIndexEvent) event);
+ handleIndexOperation((ElasticsearchIndexEvent) event);
} else if (event instanceof DeIndexOldVersionsEvent) {
- indexOperationMessage = handleDeIndexOldVersionEvent((DeIndexOldVersionsEvent) event);
+ single = handleDeIndexOldVersionEvent((DeIndexOldVersionsEvent) event);
} else {
@@ -343,9 +356,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
- // returning indexOperationMessage will send that indexOperationMessage to the index producer
// if no exception happens and the QueueMessage is returned in these results, it will get ack'd
- return new IndexEventResult(Optional.fromNullable(indexOperationMessage), Optional.of(message), thisEvent.getCreationTime());
+ return new IndexEventResult(Optional.of(single), Optional.of(message), thisEvent.getCreationTime());
} catch (IndexDocNotFoundException e){
@@ -382,6 +394,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final Entity entity, long updatedAfter) {
+ offer(new EntityIndexEvent(queueFig.getPrimaryRegion(),new EntityIdScope(applicationScope, entity.getId()), 0));
+
final EntityIndexOperation entityIndexOperation =
new EntityIndexOperation( applicationScope, entity.getId(), updatedAfter);
@@ -392,19 +406,56 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
+ private IndexOperationMessage handleEntityIndexUpdate(final QueueMessage message) {
+
+ Preconditions.checkNotNull( message, "Queue Message cannot be null for handleEntityIndexUpdate" );
+
+ final AsyncEvent event = ( AsyncEvent ) message.getBody();
+
+ Preconditions.checkNotNull(message, "QueueMessage Body cannot be null for handleEntityIndexUpdate");
+ Preconditions.checkArgument(event instanceof EntityIndexEvent, String.format("Event Type for handleEntityIndexUpdate must be ENTITY_INDEX, got %s", event.getClass()));
+
+ final EntityIndexEvent entityIndexEvent = (EntityIndexEvent) event;
+
+
+ //process the entity immediately
+ //only process the same version, otherwise ignore
+ final EntityIdScope entityIdScope = entityIndexEvent.getEntityIdScope();
+ final ApplicationScope applicationScope = entityIdScope.getApplicationScope();
+ final Id entityId = entityIdScope.getId();
+ final long updatedAfter = entityIndexEvent.getUpdatedAfter();
+
+ final EntityIndexOperation entityIndexOperation = new EntityIndexOperation( applicationScope, entityId, updatedAfter);
+
+ return eventBuilder.buildEntityIndex( entityIndexOperation ).toBlocking().lastOrDefault(null);
+ }
+
@Override
public void queueNewEdge(final ApplicationScope applicationScope,
final Entity entity,
final Edge newEdge) {
- final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
+ offer( new EdgeIndexEvent( queueFig.getPrimaryRegion(), applicationScope, entity.getId(), newEdge ));
- final IndexOperationMessage indexMessage = ecm.load( entity.getId() )
- .flatMap( loadedEntity -> eventBuilder.buildNewEdge(applicationScope, entity, newEdge) )
- .toBlocking().lastOrDefault(null);
+ }
- queueIndexOperationMessage( indexMessage );
+ private IndexOperationMessage handleEdgeIndex(final QueueMessage message) {
+
+ Preconditions.checkNotNull( message, "Queue Message cannot be null for handleEdgeIndex" );
+
+ final AsyncEvent event = (AsyncEvent) message.getBody();
+
+ Preconditions.checkNotNull( message, "QueueMessage Body cannot be null for handleEdgeIndex" );
+ Preconditions.checkArgument(event instanceof EdgeIndexEvent, String.format("Event Type for handleEdgeIndex must be EDGE_INDEX, got %s", event.getClass()));
+
+ final EdgeIndexEvent edgeIndexEvent = ( EdgeIndexEvent ) event;
+
+ final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( edgeIndexEvent.getApplicationScope() );
+
+ return ecm.load( edgeIndexEvent.getEntityId() )
+ .flatMap( loadedEntity -> eventBuilder.buildNewEdge(edgeIndexEvent.getApplicationScope(), loadedEntity, edgeIndexEvent.getEdge()) )
+ .toBlocking().lastOrDefault(null);
}
@@ -417,7 +468,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ) );
}
- public void handleEdgeDelete(final QueueMessage message) {
+ private IndexOperationMessage handleEdgeDelete(final QueueMessage message) {
Preconditions.checkNotNull( message, "Queue Message cannot be null for handleEdgeDelete" );
@@ -436,10 +487,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
logger.debug("Deleting in app scope {} with edge {}", applicationScope, edge);
}
- IndexOperationMessage indexMessage =
- eventBuilder.buildDeleteEdge(applicationScope, edge).toBlocking().lastOrDefault(null);
-
- queueIndexOperationMessage(indexMessage);
+ return eventBuilder.buildDeleteEdge(applicationScope, edge).toBlocking().lastOrDefault(null);
}
@@ -478,7 +526,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
offerTopic( elasticsearchIndexEvent );
}
- public IndexOperationMessage handleIndexOperation(final ElasticsearchIndexEvent elasticsearchIndexEvent){
+ private void handleIndexOperation(final ElasticsearchIndexEvent elasticsearchIndexEvent) throws IndexDocNotFoundException {
Preconditions.checkNotNull( elasticsearchIndexEvent, "elasticsearchIndexEvent cannot be null" );
@@ -529,7 +577,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// always do a check to ensure the indexes are initialized for the index requests
initializeEntityIndexes(indexOperationMessage);
- return indexOperationMessage;
+ // send it to to be indexed
+ indexProducer.put(indexOperationMessage).toBlocking().last();
}
@@ -599,7 +648,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
offer( new EntityDeleteEvent(queueFig.getPrimaryRegion(), new EntityIdScope( applicationScope, entityId ) ) );
}
- public void handleEntityDelete(final QueueMessage message) {
+ private IndexOperationMessage handleEntityDelete(final QueueMessage message) {
Preconditions.checkNotNull(message, "Queue Message cannot be null for handleEntityDelete");
@@ -625,14 +674,12 @@ public class AsyncEventServiceImpl implements AsyncEventService {
entityDeleteResults.getCompactedNode().toBlocking().lastOrDefault(null);
- IndexOperationMessage indexMessage = entityDeleteResults.getIndexObservable().toBlocking().lastOrDefault(null);
-
- queueIndexOperationMessage(indexMessage);
+ return entityDeleteResults.getIndexObservable().toBlocking().lastOrDefault(null);
}
- public void handleInitializeApplicationIndex(final AsyncEvent event, final QueueMessage message) {
+ private void handleInitializeApplicationIndex(final AsyncEvent event, final QueueMessage message) {
Preconditions.checkNotNull(message, "Queue Message cannot be null for handleInitializeApplicationIndex");
Preconditions.checkArgument(event instanceof InitializeApplicationIndexEvent, String.format("Event Type for handleInitializeApplicationIndex must be APPLICATION_INDEX, got %s", event.getClass()));
@@ -793,8 +840,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// collect into a list of QueueMessages that can be ack'd later
.collect(Collectors.toList());
- // sumbit the requests to Elasticsearch
- indexProducer.put(combined).toBlocking().last();
+ queueIndexOperationMessage(combined);
return queueMessages;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/10ff27cc/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/AsyncEvent.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/AsyncEvent.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/AsyncEvent.java
index 7b2b228..bd581ad 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/AsyncEvent.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/AsyncEvent.java
@@ -38,6 +38,8 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo;
@JsonIgnoreProperties( ignoreUnknown = true )
@JsonTypeInfo( use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.WRAPPER_OBJECT, property = "type" )
@JsonSubTypes( {
+ @JsonSubTypes.Type( value = EntityIndexEvent.class, name = "entityIndexEvent" ),
+ @JsonSubTypes.Type( value = EdgeIndexEvent.class, name = "edgeIndexEvent" ),
@JsonSubTypes.Type( value = EdgeDeleteEvent.class, name = "edgeDeleteEvent" ),
@JsonSubTypes.Type( value = EntityDeleteEvent.class, name = "entityDeleteEvent" ),
@JsonSubTypes.Type( value = InitializeApplicationIndexEvent.class, name = "initializeApplicationIndexEvent" ),
http://git-wip-us.apache.org/repos/asf/usergrid/blob/10ff27cc/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EdgeIndexEvent.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EdgeIndexEvent.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EdgeIndexEvent.java
new file mode 100644
index 0000000..6164dce
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EdgeIndexEvent.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.asyncevents.model;
+
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public final class EdgeIndexEvent
+ extends AsyncEvent {
+
+
+ @JsonProperty
+ protected ApplicationScope applicationScope;
+
+ @JsonProperty
+ protected Id entityId;
+
+ @JsonProperty
+ protected Edge edge;
+
+ /**
+ * Needed by jackson
+ */
+ public EdgeIndexEvent() {
+ super();
+ }
+
+ public EdgeIndexEvent(String sourceRegion, ApplicationScope applicationScope, Id entityId, Edge edge) {
+ super(sourceRegion);
+ this.applicationScope = applicationScope;
+ this.entityId = entityId;
+ this.edge = edge;
+ }
+
+
+ public ApplicationScope getApplicationScope() {
+ return applicationScope;
+ }
+
+
+ public Edge getEdge() {
+ return edge;
+ }
+
+
+ public Id getEntityId() {
+ return entityId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/10ff27cc/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityIndexEvent.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityIndexEvent.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityIndexEvent.java
new file mode 100644
index 0000000..7e8184b
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityIndexEvent.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.asyncevents.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
+
+
+public final class EntityIndexEvent extends AsyncEvent {
+
+
+ @JsonProperty
+ protected EntityIdScope entityIdScope;
+
+ @JsonProperty
+ private long updatedAfter;
+
+ public EntityIndexEvent() {
+ super();
+ }
+
+ public EntityIndexEvent(String sourceRegion, EntityIdScope entityIdScope, final long updatedAfter ) {
+ super(sourceRegion);
+ this.entityIdScope = entityIdScope;
+ this.updatedAfter = updatedAfter;
+ }
+
+
+ public long getUpdatedAfter() {
+ return updatedAfter;
+ }
+
+
+ public EntityIdScope getEntityIdScope() {
+ return entityIdScope;
+ }
+}
[19/50] usergrid git commit: Added a nested entity to gatling tests
Posted by sn...@apache.org.
Added a nested entity to gatling tests
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/44bcbc47
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/44bcbc47
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/44bcbc47
Branch: refs/heads/master
Commit: 44bcbc47e82e93a957f815e1ceeb866f63953bf8
Parents: 74de4bc
Author: George Reyes <gr...@apache.org>
Authored: Mon Apr 11 12:41:30 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Mon Apr 11 12:41:30 2016 -0700
----------------------------------------------------------------------
tests/performance/runLoadNestedEntities.sh | 98 ++++++++++++++++++++
.../datagenerators/EntityDataGenerator.scala | 40 ++++++++
.../org/apache/usergrid/enums/EntityType.scala | 3 +-
3 files changed, 140 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/44bcbc47/tests/performance/runLoadNestedEntities.sh
----------------------------------------------------------------------
diff --git a/tests/performance/runLoadNestedEntities.sh b/tests/performance/runLoadNestedEntities.sh
new file mode 100755
index 0000000..02fa71e
--- /dev/null
+++ b/tests/performance/runLoadNestedEntities.sh
@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ENTITY_WORKER_NUM= #may be overridden on command line
+#ENTITY_WORKER_COUNT= #may be overridden on command line
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #may be overridden on command line
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_SEED= #may be overridden on command line
+#RETRY_COUNT=
+#ENTITY_PROGRESS_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
+
+
+die() { echo "$@" 1>&2 ; exit 1; }
+
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [UUID_FILENAME [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]]"
+
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && UUID_FILENAME="$3"
+[ "$#" -ge 4 ] && NUM_ENTITIES="$4"
+[ "$#" -ge 5 ] && ENTITY_SEED="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_NUM="$6"
+[ "$#" -ge 7 ] && ENTITY_WORKER_COUNT="$7"
+
+shift $#
+
+SCENARIO_TYPE=loadEntities
+ENTITY_TYPE=nested
+ENTITY_PREFIX=nestedval
+
+# don't load entities as part of setup (loading entities is the point of the test)
+LOAD_ENTITIES=false
+
+#Compile everything
+mvn compile
+
+#Execute the test
+mvn gatling:execute \
+-DbaseUrl=${URL} \
+-DadminUser=${ADMIN_USER} \
+-DadminPassword=${ADMIN_PASSWORD} \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
+-DnumEntities=${NUM_ENTITIES} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
+-DentitySeed=${ENTITY_SEED} \
+-DretryCount=${RETRY_COUNT} \
+-DentityProgressCount=${ENTITY_PROGRESS_COUNT} \
+-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
+-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DentityType=${ENTITY_TYPE} \
+-DentityPrefix=${ENTITY_PREFIX} \
+-DloadEntities=${LOAD_ENTITIES} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
+-DuuidFilename=${UUID_FILENAME} \
+-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
+-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/44bcbc47/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala
----------------------------------------------------------------------
diff --git a/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala b/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala
index 326a319..b0d4f46 100755
--- a/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala
+++ b/tests/performance/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala
@@ -108,6 +108,45 @@ object EntityDataGenerator {
new JSONObject(Map("sortField" -> Utils.generateRandomInt(1,10000000))).toString()
}
+ def generateNestedEntity(name: String = null): String = {
+
+ new JSONObject(Map("magicData"->
+ new JSONObject(
+ Map(
+ "level1" -> "level1",
+ "level2" -> "level2",
+ "level3" -> "level3",
+ "level4" ->
+ new JSONObject(
+ Map(
+ "level5" -> "level5",
+ "level6" -> "level6",
+ "level7" -> "level7",
+ "level8" ->
+ new JSONObject(
+ Map(
+ "level9" -> "level9",
+ "level10" -> "level10",
+ "level11" -> "level11",
+ "level12" ->
+ new JSONObject(
+ Map(
+ "level13" -> "level13",
+ "level14" -> "level14",
+ "level15" -> "level15",
+ "level16" -> "level16"
+ )
+ )
+ )
+ )
+ )
+ )
+ )
+ )
+ )
+ ).toString()
+ }
+
val pirate1KString = "Yarrrrrrrrrrrrrrrr! Blimey! Avast ye scurvy dog! Dead men tell no tales. Hang him from the yardarm! Heave ho! Run a shot across the bow. Yo-ho-ho and a bottle of rum. Heave ho ye scallywag, or ye shall walk the plank! Weigh anchor and hoist the mizzen! Thar she blows! Shiver me timbers! Splice the mainbrace! Keelhaul the hornswaggler! Raise the Jolly Roger! Feed him to the fish! You fight like a dairy farmer. How appropriate, you fight like a cow. Batten down the hatches! Blow the man down! Swab the deck! Ahoy, matey! I'll crush ye barnacles! Fetch me grog! Gangway! Arrr, he's gone to Davy Jones's Locker. He be three sheets to the wind. Yo-ho me hearties! Prepare to be boarded! All your pieces of eight, else ye meet me cat o'nine tails! Bring 'er alongside! The rougher the seas, the smoother we sail! It's more fun to be a pirate than to join the navy. The beatings will continue until morale improves! Fifteen men on the dead man's chest! I be the captain of th
is ship -- swab the decks! "
def generateLargeMultiFieldEntity(name: String = null, entityNum: Int = 0): String = {
@@ -157,6 +196,7 @@ object EntityDataGenerator {
case EntityType.TrivialSortable => generateTrivialSortableEntity(entityName)
case EntityType.Basic => generateBasicEntity(entityName)
case EntityType.LargeMultiField => generateLargeMultiFieldEntity(entityName, entityNum)
+ case EntityType.Nested => generateNestedEntity(null)
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/44bcbc47/tests/performance/src/main/scala/org/apache/usergrid/enums/EntityType.scala
----------------------------------------------------------------------
diff --git a/tests/performance/src/main/scala/org/apache/usergrid/enums/EntityType.scala b/tests/performance/src/main/scala/org/apache/usergrid/enums/EntityType.scala
index 277cd04..c1e8ce8 100644
--- a/tests/performance/src/main/scala/org/apache/usergrid/enums/EntityType.scala
+++ b/tests/performance/src/main/scala/org/apache/usergrid/enums/EntityType.scala
@@ -20,9 +20,10 @@ object EntityType {
val Trivial = "trivial"
val TrivialSortable = "trivialSortable"
val Basic = "basic"
+ val Nested = "nested"
val LargeMultiField = "largeMultiField"
- val Values = Seq(Trivial,TrivialSortable,Basic,LargeMultiField)
+ val Values = Seq(Trivial,TrivialSortable,Basic,LargeMultiField,Nested)
def isValid(str: String): Boolean = {
Values.contains(str)
[04/50] usergrid git commit: Revert "Add flag to turn of de-indexing
of old entity versions during updates."
Posted by sn...@apache.org.
Revert "Add flag to turn of de-indexing of old entity versions during updates."
This reverts commit d7119a48c1f98b5f63a4a2fb3582f9fd1b9f9ec0.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c9d532f5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c9d532f5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c9d532f5
Branch: refs/heads/master
Commit: c9d532f5e4cabef0907b6846f620da7571532a5b
Parents: fbb5807
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 5 13:47:19 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 5 13:47:19 2016 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/corepersistence/CpEntityManager.java | 4 +---
.../org/apache/usergrid/corepersistence/EntityManagerFig.java | 4 ----
2 files changed, 1 insertion(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c9d532f5/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 42107f3..b29e6d3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -593,9 +593,7 @@ public class CpEntityManager implements EntityManager {
// queue up an event to clean-up older versions than this one from the index
- if(entityManagerFig.deIndexOldVersions()) {
- indexService.queueDeIndexOldVersion(applicationScope, entityId);
- }
+ indexService.queueDeIndexOldVersion( applicationScope, entityId );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c9d532f5/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 1528e46..45bebc6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@ -37,8 +37,4 @@ public interface EntityManagerFig extends GuicyFig {
@Key( "usergrid.entityManager_sleep_ms" )
@Default( "100" )
int sleep();
-
- @Key( "usergrid.entityManager.deindex_old_versions" )
- @Default( "false" )
- boolean deIndexOldVersions();
}
[36/50] usergrid git commit: Merge commit 'refs/pull/503/head' of
github.com:apache/usergrid into release-2.1.1
Posted by sn...@apache.org.
Merge commit 'refs/pull/503/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/99e6d406
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/99e6d406
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/99e6d406
Branch: refs/heads/master
Commit: 99e6d406cd0d60c95cf4e9e69c292b39ca3def21
Parents: 491aa08 fbced58
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Apr 15 00:47:36 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Apr 15 00:47:36 2016 +0200
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 86 ++++++++++++++------
.../collection/CollectionsResourceIT.java | 56 +++++++++++--
2 files changed, 112 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
[18/50] usergrid git commit: Fix but in app exists logic. All tests
pass with app ID cache changes.
Posted by sn...@apache.org.
Fix but in app exists logic. All tests pass with app ID cache changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9486cdd3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9486cdd3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9486cdd3
Branch: refs/heads/master
Commit: 9486cdd3e0b2aeb3739dce8cb551fb3dad132148
Parents: 177e063
Author: Dave Johnson <sn...@apache.org>
Authored: Mon Apr 11 08:42:21 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon Apr 11 08:42:21 2016 -0400
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpEntityManagerFactory.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9486cdd3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index a88bf7b..c3bcaf6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -221,7 +221,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
final UUID appId = applicationIdCache.getApplicationId( appName );
- if ( appId == null ) {
+ if ( appId != null ) {
throw new ApplicationAlreadyExistsException( name );
}
@@ -259,7 +259,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// check for pre-existing application
- if ( lookupApplication( appName ) == null ) {
+ if ( lookupApplication( appName ) != null ) {
throw new ApplicationAlreadyExistsException( appName );
}
[47/50] usergrid git commit: Reduce amount of stack traces logged by
ES exceptions and change indexing ! wildcard to "none".
Posted by sn...@apache.org.
Reduce amount of stack traces logged by ES exceptions and change indexing ! wildcard to "none".
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/dfaf344a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/dfaf344a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/dfaf344a
Branch: refs/heads/master
Commit: dfaf344a0e294520e12f672be595b52c7c857330
Parents: 06caa25
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Apr 18 11:51:31 2016 +0100
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Apr 18 11:51:31 2016 +0100
----------------------------------------------------------------------
.../usergrid/corepersistence/CpEntityManager.java | 6 +++---
.../persistence/index/impl/EsEntityIndexImpl.java | 16 ++++++++--------
.../persistence/index/impl/FailureMonitorImpl.java | 2 +-
.../collection/CollectionsResourceIT.java | 6 +++---
4 files changed, 15 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/dfaf344a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 750cf7b..73f5d5a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -627,7 +627,7 @@ public class CpEntityManager implements EntityManager {
if ( collectionIndexingSchema.isPresent()) {
Map jsonMapData = collectionIndexingSchema.get();
final ArrayList fields = (ArrayList) jsonMapData.get( "fields" );
- if ( fields.size() == 1 && fields.get(0).equals("!")) {
+ if ( fields.size() == 1 && fields.get(0).equals("none")) {
skipIndexing = true;
}
}
@@ -1819,9 +1819,9 @@ public class CpEntityManager implements EntityManager {
wildCardArrayList.add( "*" );
schemaMap.put( "fields", wildCardArrayList );
- } else if ( fieldProperties.contains( "!" )) {
+ } else if ( fieldProperties.contains( "none" )) {
ArrayList<String> wildCardArrayList = new ArrayList<>();
- wildCardArrayList.add( "!" );
+ wildCardArrayList.add( "none" );
schemaMap.put( "fields", wildCardArrayList );
} else {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/dfaf344a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index f5da29a..10ee91e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -439,7 +439,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
searchResponse = srb.execute().actionGet();
}
catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t );
+ logger.error( "Unable to communicate with Elasticsearch", t.getMessage() );
failureMonitor.fail( "Unable to execute batch", t );
throw t;
}
@@ -522,7 +522,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
}
catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t );
+ logger.error( "Unable to communicate with Elasticsearch", t.getMessage() );
failureMonitor.fail( "Unable to execute batch", t );
throw t;
}
@@ -607,7 +607,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
}
catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t );
+ logger.error( "Unable to communicate with Elasticsearch", t.getMessage() );
failureMonitor.fail( "Unable to execute batch", t );
throw t;
}
@@ -640,11 +640,11 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
@Override
public void onFailure( Throwable e ) {
- logger.error( "failed on delete index", e );
+ logger.error( "Failed on delete index", e.getMessage() );
}
} );
return Observable.from( response );
- } ).doOnError( t -> logger.error( "Failed on delete application", t ) );
+ } ).doOnError( t -> logger.error( "Failed on delete application", t.getMessage() ) );
}
@@ -762,7 +762,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
operation.doOp();
}
catch ( Exception e ) {
- logger.error( "Unable to execute operation, retrying", e );
+ logger.error( "Unable to execute operation, retrying", e.getMessage() );
try {
Thread.sleep( WAIT_TIME );
} catch ( InterruptedException ie ) {
@@ -788,7 +788,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
return Health.valueOf( chr.getStatus().name() );
}
catch ( Exception ex ) {
- logger.error( "Error connecting to ElasticSearch", ex );
+ logger.error( "Error connecting to ElasticSearch", ex.getMessage() );
}
// this is bad, red alert!
@@ -812,7 +812,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
return Health.valueOf( chr.getStatus().name() );
}
catch ( Exception ex ) {
- logger.error( "Error connecting to ElasticSearch", ex );
+ logger.error( "Error connecting to ElasticSearch", ex.getMessage() );
}
// this is bad, red alert!
http://git-wip-us.apache.org/repos/asf/usergrid/blob/dfaf344a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
index 943eabb..19749df 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/FailureMonitorImpl.java
@@ -74,7 +74,7 @@ public class FailureMonitorImpl implements FailureMonitor {
final int maxCount = indexFig.getFailRefreshCount();
if ( fails > maxCount ) {
- logger.error( "Unable to connect to elasticsearch. Reason is {}", message, throwable );
+ logger.error( "Unable to connect to elasticsearch. Reason is {}", throwable.getMessage() );
logger.warn( "We have failed to connect to Elastic Search {} times. Max allowed is {}. Resetting connection", fails, maxCount );
esProvider.releaseClient();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/dfaf344a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index b240629..8ed2154 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -943,14 +943,14 @@ public class CollectionsResourceIT extends AbstractRestIT {
/**
- * Test that when schema is "!" entity gets saved but does not get indexed
+ * Test that when schema is "none" entity gets saved but does not get indexed
*/
@Test
public void postCollectionSchemaWithWildcardIndexNone() throws Exception {
// creating schema with no index wildcard and other fields that should be ignored
ArrayList<String> indexingArray = new ArrayList<>( );
- indexingArray.add( "!" );
+ indexingArray.add( "none" );
indexingArray.add( "one" );
indexingArray.add( "two" );
Entity payload = new Entity();
@@ -965,7 +965,7 @@ public class CollectionsResourceIT extends AbstractRestIT {
Collection collection = app().collection( collectionName ).collection( "_index" ).get();
LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData();
ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" );
- assertTrue( schema.contains( "!" ) );
+ assertTrue( schema.contains( "none" ) );
assertFalse( schema.contains( "one" ) );
assertFalse( schema.contains( "two" ) );
[48/50] usergrid git commit: Fix issues with skipping entities to be
indexed.
Posted by sn...@apache.org.
Fix issues with skipping entities to be indexed.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b8f502f9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b8f502f9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b8f502f9
Branch: refs/heads/master
Commit: b8f502f9d4f0e2776b4f5c28bec66ec09cb8e12a
Parents: dfaf344
Author: Michael Russo <mr...@apigee.com>
Authored: Mon Apr 18 13:40:49 2016 +0100
Committer: Michael Russo <mr...@apigee.com>
Committed: Mon Apr 18 13:40:49 2016 +0100
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 2 +-
.../corepersistence/CpRelationManager.java | 82 +++++++++++++++++---
.../corepersistence/index/IndexServiceImpl.java | 2 +
.../service/ApplicationServiceImpl.java | 37 ++++++++-
4 files changed, 108 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b8f502f9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 73f5d5a..b74995a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -781,7 +781,7 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull(entityRef, "entityRef cannot be null");
CpRelationManager relationManager = new CpRelationManager( managerCache, indexService, collectionService,
- connectionService, this, entityManagerFig, applicationId, entityRef );
+ connectionService, this, entityManagerFig, applicationId, indexSchemaCacheFactory, entityRef );
return relationManager;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b8f502f9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 5596ab4..4082b45 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -19,7 +19,11 @@ package org.apache.usergrid.corepersistence;
import java.util.*;
+import org.apache.usergrid.corepersistence.index.IndexSchemaCache;
+import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapScope;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -75,13 +79,7 @@ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createColle
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionEdge;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchByEdge;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getNameFromEdgeType;
-import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_TITLE;
-import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY;
-import static org.apache.usergrid.persistence.Schema.TYPE_ROLE;
-import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
+import static org.apache.usergrid.persistence.Schema.*;
import static org.apache.usergrid.utils.ClassUtils.cast;
import static org.apache.usergrid.utils.InflectionUtils.singularize;
import static org.apache.usergrid.utils.MapUtils.addMapSet;
@@ -109,6 +107,8 @@ public class CpRelationManager implements RelationManager {
private final AsyncEventService indexService;
+ private final IndexSchemaCacheFactory indexSchemaCacheFactory;
+
private final CollectionService collectionService;
private final ConnectionService connectionService;
@@ -119,6 +119,7 @@ public class CpRelationManager implements RelationManager {
final ConnectionService connectionService,
final EntityManager em,
final EntityManagerFig entityManagerFig, final UUID applicationId,
+ final IndexSchemaCacheFactory indexSchemaCacheFactory,
final EntityRef headEntity) {
@@ -158,6 +159,8 @@ public class CpRelationManager implements RelationManager {
.format( "cpHeadEntity cannot be null for entity id %s, app id %s", entityId.getUuid(), applicationId ) );
this.indexService = indexService;
+ this.indexSchemaCacheFactory = indexSchemaCacheFactory;
+
}
@@ -408,10 +411,20 @@ public class CpRelationManager implements RelationManager {
//reverse
return gm.writeEdge( reverseEdge ).doOnNext( reverseEdgeWritten -> {
- indexService.queueNewEdge( applicationScope, cpHeadEntity, reverseEdge );
+
+ if ( !skipIndexingForType( cpHeadEntity.getId().getType() ) ) {
+
+ indexService.queueNewEdge(applicationScope, cpHeadEntity, reverseEdge);
+ }
+
} );
} ).doOnCompleted( () -> {
- indexService.queueNewEdge( applicationScope, memberEntity, edge );
+
+ if ( !skipIndexingForType( memberEntity.getId().getType() ) ) {
+ indexService.queueNewEdge(applicationScope, memberEntity, edge);
+ }
+
+
if ( logger.isDebugEnabled() ) {
logger.debug( "Added entity {}:{} to collection {}",
itemRef.getUuid().toString(), itemRef.getType(), collectionName );
@@ -533,8 +546,10 @@ public class CpRelationManager implements RelationManager {
//TODO: this should not happen here, needs to go to SQS
//indexProducer.put(batch).subscribe();
- indexService.queueEntityDelete(applicationScope,memberEntity.getId());
+ if ( !skipIndexingForType( memberEntity.getId().getType() ) ) {
+ indexService.queueEntityDelete(applicationScope, memberEntity.getId());
+ }
// special handling for roles collection of a group
if ( headEntity.getType().equals( Group.ENTITY_TYPE ) ) {
@@ -709,8 +724,11 @@ public class CpRelationManager implements RelationManager {
gm.writeEdge(edge).toBlocking().lastOrDefault(null); //throw an exception if this fails
- indexService.queueNewEdge( applicationScope, targetEntity, edge );
+ if ( !skipIndexingForType( targetEntity.getId().getType() ) ) {
+
+ indexService.queueNewEdge(applicationScope, targetEntity, edge);
+ }
// remove any duplicate edges (keeps the duplicate edge with same timestamp)
removeDuplicateEdgesAsync(gm, edge);
@@ -787,7 +805,14 @@ public class CpRelationManager implements RelationManager {
//delete all the edges and queue their processing
gm.loadEdgeVersions( search ).flatMap( returnedEdge -> gm.markEdge( returnedEdge ) )
- .doOnNext( returnedEdge -> indexService.queueDeleteEdge( applicationScope, returnedEdge ) ).toBlocking()
+ .doOnNext( returnedEdge -> {
+
+ if ( !skipIndexingForType( returnedEdge.getSourceNode().getType() ) || !skipIndexingForType( returnedEdge.getTargetNode().getType() ) ) {
+
+ indexService.queueDeleteEdge(applicationScope, returnedEdge);
+ }
+
+ }).toBlocking()
.lastOrDefault( null );
}
@@ -1057,4 +1082,37 @@ public class CpRelationManager implements RelationManager {
}
+ private boolean skipIndexingForType( String type ) {
+
+ boolean skipIndexing = false;
+
+ MapManager mm = getMapManagerForTypes();
+ IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm );
+ String collectionName = Schema.defaultCollectionName( type );
+ Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName );
+
+ if ( collectionIndexingSchema.isPresent()) {
+ Map jsonMapData = collectionIndexingSchema.get();
+ final ArrayList fields = (ArrayList) jsonMapData.get( "fields" );
+ if ( fields.size() == 1 && fields.get(0).equals("none")) {
+ skipIndexing = true;
+ }
+ }
+
+ return skipIndexing;
+ }
+
+ /**
+ * Get the map manager for uuid mapping
+ */
+ private MapManager getMapManagerForTypes() {
+ Id mapOwner = new SimpleId( applicationId, TYPE_APPLICATION );
+
+ final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner);
+
+ MapManager mm = managerCache.getMapManager( ms );
+
+ return mm;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b8f502f9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index ad997c8..a33453e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -226,6 +226,8 @@ public class IndexServiceImpl implements IndexService {
return Optional.absent();
}
+ fieldsToKeep.remove("none");
+
defaultProperties.addAll( fieldsToKeep );
}
else {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b8f502f9/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationServiceImpl.java
index 7c9fee3..ca2b9a2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/ApplicationServiceImpl.java
@@ -19,9 +19,12 @@
*/
package org.apache.usergrid.corepersistence.service;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
+import org.apache.usergrid.corepersistence.index.IndexSchemaCache;
+import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.Schema;
@@ -39,6 +42,9 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.utils.InflectionUtils;
import rx.Observable;
+import java.util.ArrayList;
+import java.util.Map;
+
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createGraphOperationTimestamp;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
@@ -53,6 +59,8 @@ public class ApplicationServiceImpl implements ApplicationService{
private final EventBuilder eventBuilder;
private final MapManagerFactory mapManagerFactory;
private final GraphManagerFactory graphManagerFactory;
+ private final IndexSchemaCacheFactory indexSchemaCacheFactory;
+
@Inject
@@ -61,7 +69,8 @@ public class ApplicationServiceImpl implements ApplicationService{
AsyncEventService asyncEventService,
EventBuilder eventBuilder,
MapManagerFactory mapManagerFactory,
- GraphManagerFactory graphManagerFactory
+ GraphManagerFactory graphManagerFactory,
+ IndexSchemaCacheFactory indexSchemaCacheFactory
){
this.allEntityIdsObservable = allEntityIdsObservable;
@@ -70,6 +79,7 @@ public class ApplicationServiceImpl implements ApplicationService{
this.eventBuilder = eventBuilder;
this.mapManagerFactory = mapManagerFactory;
this.graphManagerFactory = graphManagerFactory;
+ this.indexSchemaCacheFactory = indexSchemaCacheFactory;
}
@@ -119,7 +129,11 @@ public class ApplicationServiceImpl implements ApplicationService{
private Id deleteAsync(MapManager mapManager, ApplicationScope applicationScope, Id entityId ) {
try {
//Step 4 && 5
- asyncEventService.queueEntityDelete(applicationScope, entityId);
+
+ if ( !skipIndexingForType( entityId.getType(), applicationScope ) ) {
+
+ asyncEventService.queueEntityDelete(applicationScope, entityId);
+ }
//Step 6
//delete from our UUID index
mapManager.delete(entityId.getUuid().toString());
@@ -130,6 +144,25 @@ public class ApplicationServiceImpl implements ApplicationService{
}
+ private boolean skipIndexingForType( String type, ApplicationScope applicationScope ) {
+
+ boolean skipIndexing = false;
+
+ MapManager mm = getMapManagerForTypes(applicationScope);
+ IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm );
+ String collectionName = Schema.defaultCollectionName( type );
+ Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName );
+
+ if ( collectionIndexingSchema.isPresent()) {
+ Map jsonMapData = collectionIndexingSchema.get();
+ final ArrayList fields = (ArrayList) jsonMapData.get( "fields" );
+ if ( fields.size() == 1 && fields.get(0).equals("none")) {
+ skipIndexing = true;
+ }
+ }
+
+ return skipIndexing;
+ }
/**
* Get the map manager for uuid mapping
[46/50] usergrid git commit: Final changes to enhance parallel
loading of devices for push notifications.
Posted by sn...@apache.org.
Final changes to enhance parallel loading of devices for push notifications.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/06caa250
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/06caa250
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/06caa250
Branch: refs/heads/master
Commit: 06caa2509407322498c025b1b3d39135d82777cc
Parents: cc3cbfe
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Apr 17 18:02:32 2016 +0100
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Apr 17 18:02:32 2016 +0100
----------------------------------------------------------------------
.../pipeline/builder/IdBuilder.java | 2 +-
.../persistence/MultiQueryIterator.java | 2 +-
.../persistence/NotificationGraphIterator.java | 59 ++----
.../persistence/PagingResultsIterator.java | 25 ++-
.../apache/usergrid/persistence/PathQuery.java | 14 +-
.../apache/usergrid/persistence/Results.java | 4 +
.../impl/ApplicationQueueManagerImpl.java | 211 +++++--------------
7 files changed, 102 insertions(+), 215 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/06caa250/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
index 65cf7c1..781d7d5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
@@ -151,7 +151,7 @@ public class IdBuilder {
public Observable<ResultsPage<Id>> build(){
//we must add our resume filter so we drop our previous page first element if it's present
- return pipeline.withFilter( new IdFilter() ).withFilter(new ResultsPageCollector<>()).execute();
+ return pipeline.withFilter( new IdResumeFilter() ).withFilter(new ResultsPageCollector<>()).execute();
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/06caa250/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
index c5de5c1..9e28204 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/MultiQueryIterator.java
@@ -62,7 +62,7 @@ public class MultiQueryIterator implements ResultsIterator {
EntityRef ref = source.next();
Results r = getResultsFor( ref );
if ( r.size() > 0 ) {
- currentIterator = new PagingResultsIterator( r, query.getResultsLevel() );
+ currentIterator = new PagingResultsIterator( r, query.getResultsLevel(), null);
return currentIterator.hasNext();
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/06caa250/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
index a1f3246..a1b162d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
@@ -23,10 +23,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import java.util.stream.Collectors;
public class NotificationGraphIterator implements ResultsIterator, Iterable {
@@ -67,18 +63,20 @@ public class NotificationGraphIterator implements ResultsIterator, Iterable {
Object next = source.next();
Results r;
-// if(next instanceof UUID){
-//
-// UUID id = (UUID) next;
-// r = getResultsForId(id, "user");
-//
-// }else {
- EntityRef ref = (EntityRef) next;
- r = getResultsFor(ref);
- // }
+ EntityRef ref = (EntityRef) next;
+ r = getResultsFor(ref);
if (r.size() > 0) {
- currentIterator = new PagingResultsIterator(r, query.getResultsLevel());
+
+
+ if(ref.getType().equals(Group.ENTITY_TYPE)) {
+
+ currentIterator = new PagingResultsIterator(r, query.getResultsLevel(), Query.Level.REFS);
+ }else{
+ currentIterator = new PagingResultsIterator(r, query.getResultsLevel(), null);
+
+ }
+
return currentIterator.hasNext();
}
}
@@ -122,26 +120,13 @@ public class NotificationGraphIterator implements ResultsIterator, Iterable {
// if we're fetching devices through groups->users->devices, get only the IDs and don't load the entities
if( ref.getType().equals(Group.ENTITY_TYPE)){
- // query users using IDs as we don't need to load the full entities just to find their devices
- Query usersQuery = new Query();
- usersQuery.setCollection("users");
- usersQuery.setResultsLevel(Query.Level.IDS);
- usersQuery.setLimit(1000);
+ // groups->users is a passthrough to devices, load our max limit
+ query.setLimit(Query.MAX_LIMIT);
-
- // set the query level for the iterator temporarily to IDS
+ // set the query level for the when fetching users to IDS, we don't need the full entity
query.setResultsLevel(Query.Level.IDS);
- return entityManager.searchCollection(ref, usersQuery.getCollection(), usersQuery);
-
-
-// List<EntityRef> refs =
-// results.getIds().stream()
-// .map( uuid -> new SimpleEntityRef( "user", uuid) ).collect(Collectors.toList());
-//
-// // set the query level for the iterator back to REFS after mapping our IDS
-// query.setResultsLevel(Query.Level.REFS);
-// return Results.fromRefList(refs);
+ return entityManager.searchCollection(ref, "users", query);
}
@@ -151,8 +136,6 @@ public class NotificationGraphIterator implements ResultsIterator, Iterable {
devicesQuery.setCollection("devices");
devicesQuery.setResultsLevel(Query.Level.CORE_PROPERTIES);
- //query.setCollection("devices");
- //query.setResultsLevel(Query.Level.CORE_PROPERTIES);
return entityManager.searchCollection(ref, devicesQuery.getCollection(), devicesQuery);
}
@@ -177,14 +160,4 @@ public class NotificationGraphIterator implements ResultsIterator, Iterable {
}
}
-
- private Results getResultsForId(UUID uuid, String type) {
-
- EntityRef ref = new SimpleEntityRef(type, uuid);
- return getResultsFor(ref);
-
-
- }
-
-
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/06caa250/stack/core/src/main/java/org/apache/usergrid/persistence/PagingResultsIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PagingResultsIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PagingResultsIterator.java
index a883e1b..640ee06 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/PagingResultsIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PagingResultsIterator.java
@@ -19,6 +19,8 @@ package org.apache.usergrid.persistence;
import java.util.Iterator;
import java.util.List;
+import java.util.stream.Collectors;
+
import org.apache.usergrid.persistence.Query.Level;
@@ -28,20 +30,23 @@ public class PagingResultsIterator implements ResultsIterator, Iterable {
private Results results;
private Iterator currentPageIterator;
private Level level;
+ private Level overrideLevel;
public PagingResultsIterator( Results results ) {
- this( results, results.level );
+ this( results, results.level, null);
}
/**
* @param level overrides the default level from the Results - in case you want to return, say, UUIDs where the
* Query was set for Entities
+ * @param overrideLevel
*/
- public PagingResultsIterator( Results results, Level level ) {
+ public PagingResultsIterator(Results results, Level level, Level overrideLevel) {
this.results = results;
this.level = level;
+ this.overrideLevel = overrideLevel;
initCurrentPageIterator();
}
@@ -86,16 +91,32 @@ public class PagingResultsIterator implements ResultsIterator, Iterable {
*/
private boolean initCurrentPageIterator() {
List currentPage;
+ Level origLevel = level;
+ if(overrideLevel != null){
+ level=overrideLevel;
+ if(results.getIds()!=null){
+
+ List<EntityRef> userRefs = results.getIds().stream()
+ .map( uuid -> new SimpleEntityRef("user", uuid)).collect(Collectors.toList());
+
+ results.setRefs(userRefs);
+
+ }
+ }
+
if ( results != null ) {
switch ( level ) {
case IDS:
currentPage = results.getIds();
+ level = origLevel;
break;
case REFS:
currentPage = results.getRefs();
+ level = origLevel;
break;
default:
currentPage = results.getEntities();
+ level = origLevel;
}
if ( currentPage.size() > 0 ) {
currentPageIterator = currentPage.iterator();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/06caa250/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
index 215f6ac..30636ab 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/PathQuery.java
@@ -88,7 +88,7 @@ public class PathQuery<E> {
try {
if ( uuid != null && type != null ) {
- return new PagingResultsIterator( getHeadResults( em ), query.getResultsLevel() );
+ return new PagingResultsIterator( getHeadResults( em ), query.getResultsLevel(), null);
}
else {
return new MultiQueryIterator( em, source.refIterator( em, false), query );
@@ -103,7 +103,7 @@ public class PathQuery<E> {
try {
if ( uuid != null && type != null ) {
- return new PagingResultsIterator( getHeadResults( em ), query.getResultsLevel() );
+ return new PagingResultsIterator( getHeadResults( em ), query.getResultsLevel(), null);
}else {
return new NotificationGraphIterator(em, source.refIterator(em, true), query);
@@ -130,6 +130,12 @@ public class PathQuery<E> {
UUID entityId = em.getUniqueIdFromAlias( entityType, name );
+ if( entityId == null){
+ throw new
+ IllegalArgumentException("Entity with name "+name+" not found. Unable to send push notification");
+ }
+
+
return em.getEntities(Collections.singletonList(entityId), entityType);
}
@@ -143,12 +149,12 @@ public class PathQuery<E> {
if ( query.getQl() == null && query.getSingleNameOrEmailIdentifier() != null){
- return new PagingResultsIterator( getHeadResults( em ), Level.REFS );
+ return new PagingResultsIterator( getHeadResults( em ), Level.REFS, null);
}
if ( type != null && uuid != null) {
- return new PagingResultsIterator( getHeadResults( em ), Level.REFS );
+ return new PagingResultsIterator( getHeadResults( em ), Level.REFS, null);
}
else {
Query q = query;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/06caa250/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
index 2a84622..3502581 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
@@ -436,6 +436,10 @@ public class Results implements Iterable<Entity> {
level = Level.REFS;
}
+ public void setRefsOnly( List<EntityRef> resultsRefs ) {
+ refs = resultsRefs;
+ }
+
public Results withRefs( List<EntityRef> resultsRefs ) {
setRefs( resultsRefs );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/06caa250/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 1cbb2c6..2f39ae4 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -52,6 +52,8 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
private final Meter queueMeter;
private final Meter sendMeter;
+ private final static String PUSH_PROCESSING_CONCURRENCY_PROP = "usergrid.push.async.processing.concurrency";
+
HashMap<Object, ProviderAdapter> notifierHashMap; // only retrieve notifiers once
@@ -91,25 +93,22 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
return;
}
- if (logger.isTraceEnabled()) {
- logger.trace("notification {} start queuing", notification.getUuid());
- }
-
final PathQuery<Device> pathQuery = notification.getPathQuery().buildPathQuery(); //devices query
final AtomicInteger deviceCount = new AtomicInteger(); //count devices so you can make a judgement on batching
final ConcurrentLinkedQueue<String> errorMessages = new ConcurrentLinkedQueue<>(); //build up list of issues
- //get devices in querystring, and make sure you have access
+ // Get devices in querystring, and make sure you have access
if (pathQuery != null) {
final HashMap<Object, ProviderAdapter> notifierMap = getAdapterMap();
if (logger.isTraceEnabled()) {
logger.trace("notification {} start query", notification.getUuid());
}
- logger.info("notification {} start query", notification.getUuid());
+
+ logger.info("Notification {} started processing", notification.getUuid());
- // the main iterator can use graph traversal or index querying
+ // The main iterator can use graph traversal or index querying based on payload property. Default is Index.
final Iterator<Device> iterator;
if( notification.getUseGraph()){
iterator = pathQuery.graphIterator(em);
@@ -117,15 +116,24 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
iterator = pathQuery.iterator(em);
}
-// //if there are more pages (defined by PAGE_SIZE) you probably want this to be async, also if this is already a job then don't reschedule
-// if (iterator instanceof ResultsIterator && ((ResultsIterator) iterator).hasPages() && jobExecution == null) {
-// if(logger.isTraceEnabled()){
-// logger.trace("Scheduling notification job as it has multiple pages of devices.");
-// }
-// jobScheduler.scheduleQueueJob(notification, true);
-// em.update(notification);
-// return;
-// }
+ /**** Old code to scheduler large sets of data, but now the processing is fired off async in the background.
+ Leaving this only a reference of how to do it, if needed in future.
+
+ //if there are more pages (defined by PAGE_SIZE) you probably want this to be async,
+ //also if this is already a job then don't reschedule
+
+ if (iterator instanceof ResultsIterator
+ && ((ResultsIterator) iterator).hasPages() && jobExecution == null) {
+
+ if(logger.isTraceEnabled()){
+ logger.trace("Scheduling notification job as it has multiple pages of devices.");
+ }
+ jobScheduler.scheduleQueueJob(notification, true);
+ em.update(notification);
+ return;
+ }
+ ****/
+
final UUID appId = em.getApplication().getUuid();
final Map<String, Object> payloads = notification.getPayloads();
@@ -182,87 +190,57 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
};
- final Map<String, Object> filters = notification.getFilters();
+ final Map<String, Object> filters = notification.getFilters();
+ Observable processMessagesObservable = Observable.create(new IteratorObservable<EntityRef>(iterator))
- Observable processMessagesObservable = Observable.create(new IteratorObservable<UUID>(iterator))
-// .flatMap(entity -> {
-//
-// if(entity.getType().equals(Device.ENTITY_TYPE)){
-// return Observable.from(Collections.singletonList(entity));
-// }
-//
-// // if it's not a device, drill down and get them
-// return Observable.from(getDevices(entity));
-//
-// })
- .distinct()
.flatMap( entityRef -> {
return Observable.just(entityRef).flatMap(ref->{
List<Entity> entities = new ArrayList<>();
+ if( ref.getType().equals(User.ENTITY_TYPE)){
+
Query devicesQuery = new Query();
devicesQuery.setCollection("devices");
devicesQuery.setResultsLevel(Query.Level.CORE_PROPERTIES);
try {
- entities = em.searchCollection(new SimpleEntityRef("user", ref), devicesQuery.getCollection(), devicesQuery).getEntities();
+ entities = em.searchCollection(new SimpleEntityRef("user", ref.getUuid()), devicesQuery.getCollection(), devicesQuery).getEntities();
}catch (Exception e){
- logger.error("Unable to load devices for user: {}", ref);
+ logger.error("Unable to load devices for user: {}", ref.getUuid());
return Observable.empty();
}
+ }else if ( ref.getType().equals(Device.ENTITY_TYPE)){
+ try{
+ entities.add(em.get(ref));
-// if( ref.getType().equals(User.ENTITY_TYPE)){
-//
-// Query devicesQuery = new Query();
-// devicesQuery.setCollection("devices");
-// devicesQuery.setResultsLevel(Query.Level.CORE_PROPERTIES);
-//
-// try {
-//
-// entities = em.searchCollection(new SimpleEntityRef("user", ref.getUuid()), devicesQuery.getCollection(), devicesQuery).getEntities();
-//
-// }catch (Exception e){
-//
-// logger.error("Unable to load devices for user: {}", ref.getUuid());
-// return Observable.empty();
-// }
-//
-//
-// }else if ( ref.getType().equals(Device.ENTITY_TYPE)){
-//
-// try{
-// entities.add(em.get(ref));
-//
-// }catch(Exception e){
-//
-// logger.error("Unable to load device: {}", ref.getUuid());
-// return Observable.empty();
-//
-// }
-//
-// }
+ }catch(Exception e){
+
+ logger.error("Unable to load device: {}", ref.getUuid());
+ return Observable.empty();
+
+ }
+
+ }
return Observable.from(entities);
})
+ .distinct( deviceRef -> deviceRef.getUuid())
.filter( device -> {
- logger.info("Filtering device: {}", device.getUuid());
-
if(logger.isTraceEnabled()) {
logger.trace("Filtering device: {}", device.getUuid());
}
-
if(notification.getUseGraph() && filters.size() > 0 ) {
for (Map.Entry<String, Object> entry : filters.entrySet()) {
@@ -280,7 +258,6 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}
}
-
if(logger.isTraceEnabled()) {
logger.trace("Push notification filter did not match for notification {}, so removing from notification",
device.getUuid(), notification.getUuid());
@@ -321,20 +298,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}).subscribeOn(Schedulers.io());
- }, 100)
- //.map( entityRef -> entityRef.getUuid() )
- //.buffer(10)
-// .flatMap( uuids -> {
-//
-// if(logger.isTraceEnabled()) {
-// logger.trace("Processing batch of {} device(s)", uuids.size());
-// }
-//
-//
-// return Observable.from(em.getEntities(uuids, "device")).subscribeOn(Schedulers.io());
-//
-// }, 10)
-
+ }, Integer.valueOf(System.getProperty(PUSH_PROCESSING_CONCURRENCY_PROP, "50")))
.doOnError(throwable -> {
logger.error("Error while processing devices for notification : {}", notification.getUuid());
@@ -355,7 +319,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
notification.setProcessingFinished(System.currentTimeMillis());
notification.setDeviceProcessedCount(deviceCount.get());
em.update(notification);
- logger.info("{} device(s) processed for notification {}", deviceCount.get(), notification.getUuid());
+ logger.info("Notification {} finished processing {} device(s)", notification.getUuid(), deviceCount.get());
} catch (Exception e) {
logger.error("Unable to set processing finished timestamp for notification");
@@ -622,10 +586,13 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
try {
while (!subscriber.isUnsubscribed() && input.hasNext()) {
//send our input to the next
+ //logger.debug("calling next on iterator: {}", input.getClass().getSimpleName());
subscriber.onNext((T) input.next());
}
//tell the subscriber we don't have any more data
+ //logger.debug("finished iterator: {}", input.getClass().getSimpleName());
+
subscriber.onCompleted();
} catch (Throwable t) {
logger.error("failed on subscriber", t);
@@ -678,90 +645,6 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
return true;
}
- private List<EntityRef> getDevices(EntityRef ref) {
-
- List<EntityRef> devices = new ArrayList<>();
-
- final int LIMIT = Query.MID_LIMIT;
-
- try {
-
- if (User.ENTITY_TYPE.equals(ref.getType())) {
-
- UUID start = null;
- boolean initial = true;
- int resultSize = 0;
- while( initial || resultSize >= Query.DEFAULT_LIMIT) {
-
- initial = false;
-
- final List<EntityRef> mydevices = em.getCollection(ref, "devices", start, LIMIT,
- Query.Level.REFS, true).getRefs();
-
- resultSize = mydevices.size();
-
- if(mydevices.size() > 0){
- start = mydevices.get(mydevices.size() - 1 ).getUuid();
- }
-
- devices.addAll( mydevices );
-
- }
-
- } else if (Group.ENTITY_TYPE.equals(ref.getType())) {
-
- UUID start = null;
- boolean initial = true;
- int resultSize = 0;
-
- while( initial || resultSize >= LIMIT){
-
- initial = false;
-
- final List<EntityRef> myusers = em.getCollection(ref, "users", start,
- LIMIT, Query.Level.REFS, true).getRefs();
- resultSize = myusers.size();
-
- if(myusers.size() > 0){
- start = myusers.get(myusers.size() - 1 ).getUuid();
- }
-
-
- Observable.from(myusers).flatMap( user -> {
-
- try {
- devices.addAll(em.getCollection(user, "devices", null, 100,
- Query.Level.REFS, true).getRefs());
- }catch (Exception e){
- logger.error ("Unable to fetch devices for user: {}", user.getUuid());
- }
- return Observable.from(Collections.singletonList(user));
-
- }, 50).toBlocking().lastOrDefault(null);
-
-
-
-
-
- }
-
- }
- } catch (Exception e) {
-
- if (ref != null){
- logger.error("Error while retrieving devices for entity type {} and uuid {}. Error: {}",
- ref.getType(), ref.getUuid(), e);
- }else{
- logger.error("Error while retrieving devices. Entity ref was null.");
- }
-
- throw new RuntimeException("Unable to retrieve devices for EntityRef", e);
-
- }
-
- return devices;
- }
-
private String getProviderId(EntityRef device, Notifier notifier) throws Exception {
try {
[11/50] usergrid git commit: Added one-time job logging.
Posted by sn...@apache.org.
Added one-time job logging.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/021fcba6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/021fcba6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/021fcba6
Branch: refs/heads/master
Commit: 021fcba68c013414261f3e123d7ec6db8dc938fe
Parents: 55271ce
Author: Mike Dunker <md...@apigee.com>
Authored: Thu Apr 7 16:52:13 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Thu Apr 7 16:52:13 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/batch/job/OnlyOnceJob.java | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/021fcba6/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
index afbddc3..6d927b9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
+++ b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
@@ -27,6 +27,8 @@ import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.locking.Lock;
import org.apache.usergrid.locking.LockManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.annotation.PostConstruct;
@@ -40,6 +42,8 @@ import javax.annotation.PostConstruct;
@Component("OnlyOnceJob")
public abstract class OnlyOnceJob implements Job {
+ private static final Logger logger = LoggerFactory.getLogger(OnlyOnceJob.class);
+
private LockManager lockManager;
@Autowired
@@ -58,6 +62,11 @@ public abstract class OnlyOnceJob implements Job {
@PostConstruct
public void initLockManager() throws Exception {
this.lockManager = injector.getInstance(LockManager.class);
+ if (lockManager != null) {
+ logger.info("LockManager injection successful");
+ } else {
+ logger.error("LockManager injection unsuccessful");
+ }
}
/*
@@ -68,6 +77,7 @@ public abstract class OnlyOnceJob implements Job {
@Override
public void execute( JobExecution execution ) throws Exception {
+ logger.info("Executing one-time job, LockManager is {}" lockManager == null ? "null" : "not null");
String lockId = execution.getJobId().toString();
Lock lock = lockManager.createLock( emf.getManagementAppId(), String.format( "/jobs/%s", lockId ) );
[45/50] usergrid git commit: Parallel device fetching from users,
need to update to support all PN use cases with this parallelism.
Posted by sn...@apache.org.
Parallel device fetching from users, need to update to support all PN use cases with this parallelism.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cc3cbfee
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cc3cbfee
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cc3cbfee
Branch: refs/heads/master
Commit: cc3cbfee60f83e107269fe4fb558cb094b5c2032
Parents: 8cf7825
Author: Michael Russo <mr...@apigee.com>
Authored: Sun Apr 17 14:34:45 2016 +0100
Committer: Michael Russo <mr...@apigee.com>
Committed: Sun Apr 17 14:34:45 2016 +0100
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 21 ++++
.../pipeline/builder/IdBuilder.java | 6 ++
.../pipeline/read/FilterFactory.java | 8 +-
.../pipeline/read/traverse/IdFilter.java | 52 ++++++++++
.../results/IdQueryExecutor.java | 66 ++++++++++++
.../service/CollectionSearch.java | 9 ++
.../service/CollectionService.java | 5 +
.../service/CollectionServiceImpl.java | 23 ++++
.../persistence/NotificationGraphIterator.java | 79 +++++++++++++-
.../apache/usergrid/persistence/Results.java | 16 +++
.../persistence/entities/Notification.java | 16 ++-
.../impl/ApplicationQueueManagerImpl.java | 104 +++++++++++++++----
12 files changed, 369 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 9ecf466..5596ab4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.corepersistence;
import java.util.*;
+import org.apache.usergrid.corepersistence.results.IdQueryExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -614,6 +615,24 @@ public class CpRelationManager implements RelationManager {
final Optional<String> queryString = query.isGraphSearch()? Optional.<String>absent(): query.getQl();
final Id ownerId = headEntity.asId();
+
+ if(query.getLevel() == Level.IDS ){
+
+ return new IdQueryExecutor( toExecute.getCursor() ) {
+ @Override
+ protected Observable<ResultsPage<Id>> buildNewResultsPage(
+ final Optional<String> cursor ) {
+
+ final CollectionSearch search =
+ new CollectionSearch( applicationScope, ownerId, collectionName, collection.getType(), toExecute.getLimit(),
+ queryString, cursor );
+
+ return collectionService.searchCollectionIds( search );
+ }
+ }.next();
+
+ }
+
//wire the callback so we can get each page
return new EntityQueryExecutor( toExecute.getCursor() ) {
@Override
@@ -989,6 +1008,8 @@ public class CpRelationManager implements RelationManager {
// }
// }
+
+
return query;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
index 0f784a6..65cf7c1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
@@ -30,6 +30,7 @@ import org.apache.usergrid.corepersistence.pipeline.read.collect.ConnectionRefRe
import org.apache.usergrid.corepersistence.pipeline.read.collect.IdResumeFilter;
import org.apache.usergrid.corepersistence.pipeline.read.collect.ResultsPageCollector;
import org.apache.usergrid.corepersistence.pipeline.read.search.Candidate;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.IdFilter;
import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -148,4 +149,9 @@ public class IdBuilder {
return new ConnectionRefBuilder(connectionRefFilter);
}
+ public Observable<ResultsPage<Id>> build(){
+ //we must add our resume filter so we drop our previous page first element if it's present
+ return pipeline.withFilter( new IdFilter() ).withFilter(new ResultsPageCollector<>()).execute();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
index ca5695c..883fdc8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
@@ -27,13 +27,7 @@ import org.apache.usergrid.corepersistence.pipeline.read.search.CandidateEntityF
import org.apache.usergrid.corepersistence.pipeline.read.search.CandidateIdFilter;
import org.apache.usergrid.corepersistence.pipeline.read.search.SearchCollectionFilter;
import org.apache.usergrid.corepersistence.pipeline.read.search.SearchConnectionFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.traverse.EntityIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.traverse.EntityLoadVerifyFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphCollectionByIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphCollectionFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphConnectionByIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphConnectionByTypeFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphConnectionFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.*;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/IdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/IdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/IdFilter.java
new file mode 100644
index 0000000..9b3a05a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/IdFilter.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+
+import java.util.List;
+
+
+/**
+ * This command is a stopgap to make migrating 1.0 code easier. Once full traversal has been implemented, this should
+ * be removed
+ */
+public class IdFilter extends AbstractFilter<FilterResult<Id>, FilterResult<Id>>{
+
+ @Inject
+ public IdFilter() {};
+
+
+
+ @Override
+ public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterValueObservable ) {
+ //ignore what our input was, and simply emit the id specified
+ return filterValueObservable.map( idFilterResult -> new FilterResult( idFilterResult.getValue(), idFilterResult.getPath() ));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdQueryExecutor.java
new file mode 100644
index 0000000..5a0bb3f
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdQueryExecutor.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * Processes our results of connection refs
+ */
+@Deprecated//Required for 1.0 compatibility
+public abstract class IdQueryExecutor extends ObservableQueryExecutor<Id> {
+
+
+ protected IdQueryExecutor( final Optional<String> startCursor ) {
+ super( startCursor );
+ }
+
+
+ @Override
+ protected Results createResults( final ResultsPage resultsPage ) {
+ final List<Id> ids = resultsPage.getEntityList();
+
+ List<UUID> uuids = ids.stream().map(id -> id.getUuid()).collect(Collectors.toList());
+
+ final Results results = Results.fromIdList(uuids);
+
+ return results;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java
index ab8a8bc..602a5b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionSearch.java
@@ -29,6 +29,10 @@ import com.google.common.base.Optional;
*/
public class CollectionSearch {
+ public enum Level {
+ IDS, ALL
+ }
+
private final ApplicationScope applicationScope;
private final Id collectionOwnerId;
private final String collectionName;
@@ -36,6 +40,7 @@ public class CollectionSearch {
private final int limit;
private final Optional<String> query;
private final Optional<String> cursor;
+ private Level level = Level.ALL;
public CollectionSearch( final ApplicationScope applicationScope, final Id collectionOwnerId, final String
@@ -84,4 +89,8 @@ public class CollectionSearch {
public Id getCollectionOwnerId() {
return collectionOwnerId;
}
+
+ public void setResultsLevel(Level level){ this.level = level; }
+
+ public Level getResultsLevel(){ return level; }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionService.java
index eef741a..6a46022 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionService.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.corepersistence.service;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
@@ -35,4 +36,8 @@ public interface CollectionService {
* @return An observable with results page entries for the stream
*/
Observable<ResultsPage<Entity>> searchCollection(final CollectionSearch search);
+
+
+
+ Observable<ResultsPage<Id>> searchCollectionIds(final CollectionSearch search);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java
index fa79d09..9244315 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/service/CollectionServiceImpl.java
@@ -29,6 +29,7 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
@@ -73,4 +74,26 @@ public class CollectionServiceImpl implements CollectionService {
return results.build();
}
+
+ @Override
+ public Observable<ResultsPage<Id>> searchCollectionIds(final CollectionSearch search ) {
+
+
+ final ApplicationScope applicationScope = search.getApplicationScope();
+ final String collectionName = search.getCollectionName();
+ final Optional<String> query = search.getQuery();
+
+ final IdBuilder pipelineBuilder =
+ pipelineBuilderFactory.create( applicationScope ).withCursor( search.getCursor() )
+ .withLimit( search.getLimit() ).fromId( search.getCollectionOwnerId() );
+
+
+ final IdBuilder results;
+
+
+ results = pipelineBuilder.traverseCollection( collectionName );
+
+
+ return results.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java b/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
index b83f555..a1f3246 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/NotificationGraphIterator.java
@@ -17,10 +17,16 @@
package org.apache.usergrid.persistence;
+import org.apache.usergrid.persistence.entities.Group;
+import org.apache.usergrid.persistence.entities.User;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
public class NotificationGraphIterator implements ResultsIterator, Iterable {
@@ -58,8 +64,19 @@ public class NotificationGraphIterator implements ResultsIterator, Iterable {
return true;
}
while (source.hasNext()) {
- EntityRef ref = source.next();
- Results r = getResultsFor(ref);
+ Object next = source.next();
+ Results r;
+
+// if(next instanceof UUID){
+//
+// UUID id = (UUID) next;
+// r = getResultsForId(id, "user");
+//
+// }else {
+ EntityRef ref = (EntityRef) next;
+ r = getResultsFor(ref);
+ // }
+
if (r.size() > 0) {
currentIterator = new PagingResultsIterator(r, query.getResultsLevel());
return currentIterator.hasNext();
@@ -90,13 +107,57 @@ public class NotificationGraphIterator implements ResultsIterator, Iterable {
try {
+
+ query.setLimit(Query.MAX_LIMIT); // always fetch our MAX limit to reduce # of IO hops
if (query.getCollection() != null) {
+ // make sure this results in graph traversal
+ query.setQl("select *");
+
if(logger.isTraceEnabled()) {
logger.trace("Fetching with refType: {}, collection: {} with no query",
ref.getType(), query.getCollection());
}
- return entityManager.searchCollection(ref, query.getCollection(), null);
+
+ // if we're fetching devices through groups->users->devices, get only the IDs and don't load the entities
+ if( ref.getType().equals(Group.ENTITY_TYPE)){
+
+ // query users using IDs as we don't need to load the full entities just to find their devices
+ Query usersQuery = new Query();
+ usersQuery.setCollection("users");
+ usersQuery.setResultsLevel(Query.Level.IDS);
+ usersQuery.setLimit(1000);
+
+
+ // set the query level for the iterator temporarily to IDS
+ query.setResultsLevel(Query.Level.IDS);
+
+ return entityManager.searchCollection(ref, usersQuery.getCollection(), usersQuery);
+
+
+// List<EntityRef> refs =
+// results.getIds().stream()
+// .map( uuid -> new SimpleEntityRef( "user", uuid) ).collect(Collectors.toList());
+//
+// // set the query level for the iterator back to REFS after mapping our IDS
+// query.setResultsLevel(Query.Level.REFS);
+// return Results.fromRefList(refs);
+
+ }
+
+ if( ref.getType().equals(User.ENTITY_TYPE)){
+
+ Query devicesQuery = new Query();
+ devicesQuery.setCollection("devices");
+ devicesQuery.setResultsLevel(Query.Level.CORE_PROPERTIES);
+
+ //query.setCollection("devices");
+ //query.setResultsLevel(Query.Level.CORE_PROPERTIES);
+ return entityManager.searchCollection(ref, devicesQuery.getCollection(), devicesQuery);
+ }
+
+ return entityManager.searchCollection(ref, query.getCollection(), query);
+
} else {
@@ -105,7 +166,7 @@ public class NotificationGraphIterator implements ResultsIterator, Iterable {
ref.getType(), query.getCollection());
}
- query.setQl("select *");
+ query.setQl("select *"); // make sure this results in graph traversal
return entityManager.searchTargetEntities(ref, query);
}
@@ -116,4 +177,14 @@ public class NotificationGraphIterator implements ResultsIterator, Iterable {
}
}
+
+ private Results getResultsForId(UUID uuid, String type) {
+
+ EntityRef ref = new SimpleEntityRef(type, uuid);
+ return getResultsFor(ref);
+
+
+ }
+
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
index e9a3251..2a84622 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Results.java
@@ -708,6 +708,22 @@ public class Results implements Iterable<Entity> {
}
}
+ public void addEntities( Results results){
+
+ if(entities == null){
+ //init();
+ entities = new ArrayList<>();
+ level = Level.CORE_PROPERTIES;
+ }
+
+ if( results.getEntities().size() > 0){
+
+ entities.addAll(results.getEntities());
+
+ }
+
+ }
+
/** Remove the passed in results from the current results */
public void subtract( Results results ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
index aca10cf..d4f3529 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
@@ -409,20 +409,28 @@ public class Notification extends TypedEntity {
if ( pathTokens.size() == 1 && collection.equals(InflectionUtils.pluralize(Group.ENTITY_TYPE) )){
- Query usersQuery = new Query();
+ final Query usersQuery = new Query();
usersQuery.setQl("select *");
usersQuery.setCollection("users");
usersQuery.setLimit(100);
- Query devicesQuery = new Query();
+ final Query devicesQuery = new Query();
devicesQuery.setQl("select *");
devicesQuery.setCollection("devices");
- usersQuery.setLimit(100);
+ devicesQuery.setLimit(100);
// build up the chain so the proper iterators can be used later
- pathQuery = pathQuery.chain( usersQuery ).chain( devicesQuery );
+ pathQuery = pathQuery.chain( usersQuery );//.chain( devicesQuery );
+
+ }else if(pathTokens.size() == 1 && collection.equals(InflectionUtils.pluralize(User.ENTITY_TYPE))){
+
+ final Query devicesQuery = new Query();
+ devicesQuery.setQl("select *");
+ devicesQuery.setCollection("devices");
+ devicesQuery.setLimit(100);
+ pathQuery = pathQuery.chain( devicesQuery );
}
} else {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cc3cbfee/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 5ce1b93..1cbb2c6 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -19,10 +19,7 @@ package org.apache.usergrid.services.notifications.impl;
import com.codahale.metrics.Meter;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.persistence.*;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.*;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.queue.QueueManager;
@@ -108,6 +105,8 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
if (logger.isTraceEnabled()) {
logger.trace("notification {} start query", notification.getUuid());
}
+ logger.info("notification {} start query", notification.getUuid());
+
// the main iterator can use graph traversal or index querying
@@ -185,30 +184,80 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
final Map<String, Object> filters = notification.getFilters();
- Observable processMessagesObservable = Observable.create(new IteratorObservable<Entity>(iterator))
- .flatMap(entity -> {
- if(entity.getType().equals(Device.ENTITY_TYPE)){
- return Observable.from(Collections.singletonList(entity));
- }
- // if it's not a device, drill down and get them
- return Observable.from(getDevices(entity));
+ Observable processMessagesObservable = Observable.create(new IteratorObservable<UUID>(iterator))
+// .flatMap(entity -> {
+//
+// if(entity.getType().equals(Device.ENTITY_TYPE)){
+// return Observable.from(Collections.singletonList(entity));
+// }
+//
+// // if it's not a device, drill down and get them
+// return Observable.from(getDevices(entity));
+//
+// })
+ .distinct()
+ .flatMap( entityRef -> {
+
+ return Observable.just(entityRef).flatMap(ref->{
+
+ List<Entity> entities = new ArrayList<>();
+
+ Query devicesQuery = new Query();
+ devicesQuery.setCollection("devices");
+ devicesQuery.setResultsLevel(Query.Level.CORE_PROPERTIES);
+
+ try {
+
+ entities = em.searchCollection(new SimpleEntityRef("user", ref), devicesQuery.getCollection(), devicesQuery).getEntities();
+
+ }catch (Exception e){
+
+ logger.error("Unable to load devices for user: {}", ref);
+ return Observable.empty();
+ }
- })
- .distinct(ref -> ref.getUuid() )
- .map( entityRef -> entityRef.getUuid() )
- .buffer(10)
- .flatMap( uuids -> {
- if(logger.isTraceEnabled()) {
- logger.trace("Processing batch of {} device(s)", uuids.size());
- }
- return Observable.from(em.getEntities(uuids, "device"))
+// if( ref.getType().equals(User.ENTITY_TYPE)){
+//
+// Query devicesQuery = new Query();
+// devicesQuery.setCollection("devices");
+// devicesQuery.setResultsLevel(Query.Level.CORE_PROPERTIES);
+//
+// try {
+//
+// entities = em.searchCollection(new SimpleEntityRef("user", ref.getUuid()), devicesQuery.getCollection(), devicesQuery).getEntities();
+//
+// }catch (Exception e){
+//
+// logger.error("Unable to load devices for user: {}", ref.getUuid());
+// return Observable.empty();
+// }
+//
+//
+// }else if ( ref.getType().equals(Device.ENTITY_TYPE)){
+//
+// try{
+// entities.add(em.get(ref));
+//
+// }catch(Exception e){
+//
+// logger.error("Unable to load device: {}", ref.getUuid());
+// return Observable.empty();
+//
+// }
+//
+// }
+ return Observable.from(entities);
+
+ })
.filter( device -> {
+ logger.info("Filtering device: {}", device.getUuid());
+
if(logger.isTraceEnabled()) {
logger.trace("Filtering device: {}", device.getUuid());
}
@@ -233,7 +282,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}
if(logger.isTraceEnabled()) {
- logger.trace("Push notification filter matched for notification {}, so removing from notification",
+ logger.trace("Push notification filter did not match for notification {}, so removing from notification",
device.getUuid(), notification.getUuid());
}
return false;
@@ -271,7 +320,20 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}).subscribeOn(Schedulers.io());
- }, 10)
+
+ }, 100)
+ //.map( entityRef -> entityRef.getUuid() )
+ //.buffer(10)
+// .flatMap( uuids -> {
+//
+// if(logger.isTraceEnabled()) {
+// logger.trace("Processing batch of {} device(s)", uuids.size());
+// }
+//
+//
+// return Observable.from(em.getEntities(uuids, "device")).subscribeOn(Schedulers.io());
+//
+// }, 10)
.doOnError(throwable -> {
[27/50] usergrid git commit: Set a processingFinished timestamp so we
can tell how long it takes to queue up push notifications for a particular
query.
Posted by sn...@apache.org.
Set a processingFinished timestamp so we can tell how long it takes to queue up push notifications for a particular query.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8c0338cf
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8c0338cf
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8c0338cf
Branch: refs/heads/master
Commit: 8c0338cf664778f41b04ad46805eaecffea84852
Parents: 1127480
Author: Michael Russo <mr...@apigee.com>
Authored: Wed Apr 13 16:08:25 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Wed Apr 13 16:08:25 2016 +0200
----------------------------------------------------------------------
.../usergrid/persistence/entities/Notification.java | 13 +++++++++++++
.../impl/ApplicationQueueManagerImpl.java | 10 ++++++++++
2 files changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8c0338cf/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
index dc7c989..1350593 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Notification.java
@@ -57,6 +57,10 @@ public class Notification extends TypedEntity {
@EntityProperty
protected Long queued;
+ /** Timestamp (ms) when the notification was processed */
+ @EntityProperty
+ protected Long processingFinished;
+
/** Timestamp (ms) when send notification started */
@EntityProperty
protected Long started;
@@ -138,6 +142,15 @@ public class Notification extends TypedEntity {
}
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ public Long getProcessingFinished() {
+ return processingFinished;
+ }
+
+ public void setProcessingFinished(Long processingFinished) {
+ this.processingFinished = processingFinished;
+ }
+
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public Long getDeliver() {
return deliver;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8c0338cf/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index f819e39..0b0432d 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -204,6 +204,16 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
})
+ .doOnCompleted( () -> {
+
+ try {
+ notification.setProcessingFinished(System.currentTimeMillis());
+ em.update(notification);
+ } catch (Exception e) {
+ logger.error("Unable to set processing finished timestamp for notification");
+ }
+
+ })
.doOnError(throwable -> logger.error("Failed while trying to send notification", throwable));
//TODO verify error handling here
[02/50] usergrid git commit: Fix issue with addSort being appended to
delete queries many times when there are multiple versions of an entity.
Posted by sn...@apache.org.
Fix issue with addSort being appended to delete queries many times when there are multiple versions of an entity.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d3df2393
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d3df2393
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d3df2393
Branch: refs/heads/master
Commit: d3df23930218703cda5ea9e7fea86a52ba7efb4a
Parents: a6cfda8
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 5 13:45:42 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 5 13:45:42 2016 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d3df2393/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 111e16b..98c0738 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -471,7 +471,9 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
final QueryBuilder entityQuery = QueryBuilders
.termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(edge.getNodeId()));
- final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
+ final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder()
+ .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC);
+
if ( logger.isDebugEnabled() ) {
logger.debug( "Searching for edges in (read alias): {}\n nodeId: {},\n query: {} ",
@@ -496,7 +498,6 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
searchResponse = srb
.setQuery(finalQuery)
.setSize(searchLimit)
- .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC)
.execute()
.actionGet();
@@ -553,7 +554,8 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
final QueryBuilder nodeQuery = QueryBuilders
.termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(entityId));
- final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder();
+ final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder()
+ .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC);
try {
@@ -580,7 +582,6 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
searchResponse = srb
.setQuery(finalQuery)
.setSize(searchLimit)
- .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC)
.execute()
.actionGet();
[12/50] usergrid git commit: fix typo
Posted by sn...@apache.org.
fix typo
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b8557287
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b8557287
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b8557287
Branch: refs/heads/master
Commit: b855728750023e795e4e737d698d1fd94d6b9282
Parents: 021fcba
Author: Mike Dunker <md...@apigee.com>
Authored: Thu Apr 7 17:03:55 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Thu Apr 7 17:03:55 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b8557287/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
index 6d927b9..ac56c7b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
+++ b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
@@ -77,7 +77,7 @@ public abstract class OnlyOnceJob implements Job {
@Override
public void execute( JobExecution execution ) throws Exception {
- logger.info("Executing one-time job, LockManager is {}" lockManager == null ? "null" : "not null");
+ logger.info("Executing one-time job, LockManager is {}", lockManager == null ? "null" : "not null");
String lockId = execution.getJobId().toString();
Lock lock = lockManager.createLock( emf.getManagementAppId(), String.format( "/jobs/%s", lockId ) );
[31/50] usergrid git commit: Added code fix for 500 errors when not
using a token. Now accepts client credentials as well.
Posted by sn...@apache.org.
Added code fix for 500 errors when not using a token. Now accepts client credentials as well.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3a442b0b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3a442b0b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3a442b0b
Branch: refs/heads/master
Commit: 3a442b0bb580e6917ece785ffae5e9b47484f092
Parents: a462244
Author: George Reyes <gr...@apache.org>
Authored: Wed Apr 13 12:59:45 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Wed Apr 13 12:59:45 2016 -0700
----------------------------------------------------------------------
.../collection/CollectionsResourceIT.java | 40 ++++++++++++++++++++
.../rest/test/resource/ClientSetup.java | 10 +++++
.../services/AbstractCollectionService.java | 23 +++++++++--
3 files changed, 70 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3a442b0b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index e0b75ba..d052564 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -25,6 +25,7 @@ import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.rest.test.resource.AbstractRestIT;
import org.apache.usergrid.rest.test.resource.model.ApiResponse;
import org.apache.usergrid.rest.test.resource.model.Collection;
+import org.apache.usergrid.rest.test.resource.model.Credentials;
import org.apache.usergrid.rest.test.resource.model.Entity;
import org.apache.usergrid.rest.test.resource.model.QueryParameters;
import org.apache.usergrid.rest.test.resource.model.Token;
@@ -37,6 +38,9 @@ import org.slf4j.LoggerFactory;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.ClientErrorException;
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.core.MediaType;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -119,6 +123,42 @@ public class CollectionsResourceIT extends AbstractRestIT {
}
@Test
+ public void postToCollectionSchemaUsingOrgAppCreds(){
+ //Creating schema.
+ //this could be changed to a hashmap.
+ ArrayList<String> indexingArray = new ArrayList<>( );
+
+
+ //field "fields" is required.
+ Entity payload = new Entity();
+ payload.put( "fields", indexingArray);
+
+
+ Credentials appCredentials = clientSetup.getAppCredentials();
+
+
+ try {
+
+ this.pathResource( getOrgAppPath( "testcollections/_indexes" ) ).post( false, payload,
+ new QueryParameters().addParam( "grant_type", "client_credentials" ).addParam( "client_id",
+ String.valueOf( ( ( Map ) appCredentials.get( "credentials" ) ).get( "client_id" ) ) )
+ .addParam( "client_secret", String.valueOf(
+ ( ( Map ) appCredentials.get( "credentials" ) ).get( "client_secret" ) ) ) );
+ }catch(Exception e){
+ fail("This should return a success.");
+ }
+
+ refreshIndex();
+
+
+ Collection collection = this.app().collection( "testCollections" ).collection( "_index" ).get();
+
+ LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData();
+ assertEquals( "app credentials",testCollectionSchema.get( "lastUpdateBy" ) );
+ assertEquals( 0,testCollectionSchema.get( "lastReindexed" ) );
+ }
+
+ @Test
public void deleteCollectionSchema() throws Exception {
//Creating schema.
//this could be changed to a hashmap.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3a442b0b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/ClientSetup.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/ClientSetup.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/ClientSetup.java
index 2a13880..38a7e12 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/ClientSetup.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/ClientSetup.java
@@ -49,6 +49,7 @@ public class ClientSetup implements TestRule {
protected String superuserName = "superuser";
protected String superuserPassword = "superpassword";
protected Credentials clientCredentials;
+ protected Credentials appCredentials;
protected Organization organization;
protected Entity application;
@@ -108,11 +109,16 @@ public class ClientSetup implements TestRule {
restClient.management().token().get(username, password);
clientCredentials = restClient.management().orgs().org(orgName).credentials().get(Credentials.class);
+ //appCredentials = restClient.management().orgs().org(orgName).app().path//.credentials().get(Credentials.class);
+
ApiResponse appResponse = restClient.management().orgs()
.org(organization.getName()).app().post(new Application(appName));
appUuid = (String) appResponse.getEntities().get(0).get("uuid");
application = new Application(appResponse);
+
+ appCredentials = restClient.pathResource( "management/orgs/"+orgName+"/apps/"+appName+"/credentials" ).get( Credentials.class,true );
+
refreshIndex();
ApiResponse response = restClient.management().token().post(new Token(username, password));
@@ -163,6 +169,10 @@ public class ClientSetup implements TestRule {
return clientCredentials;
}
+ public Credentials getAppCredentials(){
+ return appCredentials;
+ }
+
public void refreshIndex() {
this.restClient.refreshIndex(getOrganizationName(), getAppName(),
CpNamingUtils.getManagementApplicationId().getUuid().toString());
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3a442b0b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
index 8b23e7a..c5473cf 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/AbstractCollectionService.java
@@ -38,6 +38,8 @@ import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
import org.apache.usergrid.persistence.Query.Level;
import org.apache.usergrid.security.shiro.principals.AdminUserPrincipal;
+import org.apache.usergrid.security.shiro.principals.ApplicationPrincipal;
+import org.apache.usergrid.security.shiro.principals.PrincipalIdentifier;
import org.apache.usergrid.security.shiro.utils.SubjectUtils;
import org.apache.usergrid.services.ServiceResults.Type;
import org.apache.usergrid.services.exceptions.ForbiddenServiceOperationException;
@@ -334,11 +336,26 @@ public class AbstractCollectionService extends AbstractService {
ServiceContext context = serviceRequest.getAppContext();
checkPermissionsForCollection( context );
+ //TODO: write rest test for these line of codes
Subject currentUser = SubjectUtils.getSubject();
- Object currentUser2 =currentUser.getPrincipal();
+ Object currentUserPrincipal =currentUser.getPrincipal();
- Map collectionSchema = em.createCollectionSchema(context.getCollectionName(),
- ( ( AdminUserPrincipal ) currentUser2 ).getUser().getEmail(),context.getProperties());
+ Map collectionSchema = null;
+
+ if(currentUserPrincipal instanceof AdminUserPrincipal) {
+ AdminUserPrincipal adminUserPrincipal = ( AdminUserPrincipal ) currentUserPrincipal;
+
+ collectionSchema = em.createCollectionSchema( context.getCollectionName(),
+ adminUserPrincipal.getUser().getEmail(), context.getProperties() );
+ }
+ else if(currentUserPrincipal instanceof ApplicationPrincipal){
+ collectionSchema = em.createCollectionSchema( context.getCollectionName(),
+ "app credentials", context.getProperties() );
+ }
+ else if ( currentUserPrincipal instanceof PrincipalIdentifier ) {
+ collectionSchema = em.createCollectionSchema( context.getCollectionName(),
+ "generic credentials", context.getProperties() );
+ }
return new ServiceResults( this, context, Type.COLLECTION, Results.fromData( collectionSchema ), null, null );
[20/50] usergrid git commit: Fix issues with notification segmenting
to ensure it's more efficient.
Posted by sn...@apache.org.
Fix issues with notification segmenting to ensure it's more efficient.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c1375bf6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c1375bf6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c1375bf6
Branch: refs/heads/master
Commit: c1375bf6caa60f36038571c3cd0c2a0b719e36f5
Parents: 8e4d7ee
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 12 00:25:14 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 12 00:25:14 2016 +0200
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 14 +++++
.../org/apache/usergrid/persistence/Query.java | 2 +
.../services/notifications/QueueListener.java | 41 ++++++------
.../impl/ApplicationQueueManagerImpl.java | 65 +++++++++++---------
.../usergrid/services/queues/QueueListener.java | 2 +-
.../apns/NotificationsServiceIT.java | 6 +-
.../gcm/NotificationsServiceIT.java | 1 -
7 files changed, 72 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c1375bf6/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 8b0174c..5cd7c7a 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -447,6 +447,20 @@ usergrid.scheduler.job.workers=4
usergrid.scheduler.job.queueName=/jobs
+############################### Usergrid Push Notifications #############################
+#
+# Usergrid processes individual push notifications asynchronously using a queue. Below are
+# settings that can be used to tune this processing.
+
+
+# Set the number of queue consumers to read from the in-region push notification queue.
+#
+usergrid.push.worker_count=8
+
+# Set the sleep time between queue polling ( in milliseconds)
+#
+usergrid.push.sleep=100
+
############################### Usergrid Central SSO #############################
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c1375bf6/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 52e3b4e..150a1b0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -61,6 +61,8 @@ public class Query {
public static final int DEFAULT_LIMIT = 10;
+ public static final int MID_LIMIT = 500;
+
public static final int MAX_LIMIT = 1000;
public static final String PROPERTY_UUID = "uuid";
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c1375bf6/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index de9cf06..55d1491 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -45,10 +45,10 @@ import java.util.concurrent.atomic.AtomicLong;
* Singleton listens for notifications queue messages
*/
public class QueueListener {
- public final int MESSAGE_TRANSACTION_TIMEOUT = 25 * 1000;
+
private final QueueManagerFactory queueManagerFactory;
- public long DEFAULT_SLEEP = 5000;
+ public static long DEFAULT_SLEEP = 100;
private static final Logger logger = LoggerFactory.getLogger(QueueListener.class);
@@ -61,9 +61,6 @@ public class QueueListener {
private Properties properties;
-
- private ServiceManager svcMgr;
-
private long sleepWhenNoneFound = 0;
private long sleepBetweenRuns = 0;
@@ -71,8 +68,8 @@ public class QueueListener {
private ExecutorService pool;
private List<Future> futures;
- public final int MAX_THREADS = 2;
- private Integer batchSize = 10;
+ private static final int PUSH_CONSUMER_MAX_THREADS = 8;
+ public static final int MAX_TAKE = 10;
private String queueName;
private int consecutiveCallsToRemoveDevices;
@@ -99,15 +96,14 @@ public class QueueListener {
try {
- sleepBetweenRuns = new Long(properties.getProperty("usergrid.notifications.listener.sleep.between", ""+sleepBetweenRuns)).longValue();
- sleepWhenNoneFound = new Long(properties.getProperty("usergrid.notifications.listener.sleep.after", ""+DEFAULT_SLEEP)).longValue();
- batchSize = new Integer(properties.getProperty("usergrid.notifications.listener.batchSize", (""+batchSize)));
+ sleepBetweenRuns = new Long(properties.getProperty("usergrid.push.sleep", "" + DEFAULT_SLEEP));
+ sleepWhenNoneFound = new Long(properties.getProperty("usergrid.push.sleep", "" + DEFAULT_SLEEP));
consecutiveCallsToRemoveDevices = new Integer(properties.getProperty("usergrid.notifications.inactive.interval", ""+200));
queueName = ApplicationQueueManagerImpl.getQueueNames(properties);
- int maxThreads = new Integer(properties.getProperty("usergrid.notifications.listener.maxThreads", ""+MAX_THREADS));
+ int maxThreads = new Integer(properties.getProperty("usergrid.push.worker_count", ""+PUSH_CONSUMER_MAX_THREADS));
- futures = new ArrayList<Future>(maxThreads);
+ futures = new ArrayList<>(maxThreads);
//create our thread pool based on our threadcount.
@@ -144,33 +140,40 @@ public class QueueListener {
}
private void execute(int threadNumber){
+
if(Thread.currentThread().isDaemon()) {
Thread.currentThread().setDaemon(true);
}
+
Thread.currentThread().setName(getClass().getSimpleName()+"_PushNotifications-"+threadNumber);
final AtomicInteger consecutiveExceptions = new AtomicInteger();
+
if (logger.isTraceEnabled()) {
logger.trace("QueueListener: Starting execute process.");
}
+
Meter meter = metricsService.getMeter(QueueListener.class, "execute.commit");
com.codahale.metrics.Timer timer = metricsService.getTimer(QueueListener.class, "execute.dequeue");
- svcMgr = smf.getServiceManager(smf.getManagementAppId());
+
if (logger.isTraceEnabled()) {
logger.trace("getting from queue {} ", queueName);
}
+
QueueScope queueScope = new QueueScopeImpl( queueName, QueueScope.RegionImplementation.LOCAL);
QueueManager queueManager = queueManagerFactory.getQueueManager(queueScope);
+
// run until there are no more active jobs
final AtomicLong runCount = new AtomicLong(0);
+
//cache to retrieve push manager, cached per notifier, so many notifications will get same push manager
LoadingCache<UUID, ApplicationQueueManager> queueManagerMap = getQueueManagerCache(queueManager);
while ( true ) {
Timer.Context timerContext = timer.time();
- rx.Observable.from(queueManager.getMessages(getBatchSize(), ApplicationQueueMessage.class))
- .buffer(getBatchSize())
+ rx.Observable.from(queueManager.getMessages(MAX_TAKE, ApplicationQueueMessage.class))
+ .buffer(MAX_TAKE)
.doOnNext(messages -> {
try {
@@ -329,12 +332,4 @@ public class QueueListener {
pool.shutdownNow();
}
-
- public void setBatchSize(int batchSize){
- this.batchSize = batchSize;
- }
- public int getBatchSize(){return batchSize;}
-
-
-
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c1375bf6/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 04e60b7..6c28d2f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -120,7 +120,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
final UUID appId = em.getApplication().getUuid();
final Map<String, Object> payloads = notification.getPayloads();
- final Func1<EntityRef, ApplicationQueueMessage> sendMessageFunction = deviceRef -> {
+ final Func1<EntityRef, Optional<ApplicationQueueMessage>> sendMessageFunction = deviceRef -> {
try {
long now = System.currentTimeMillis();
@@ -145,7 +145,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
if (notifierId == null) {
//TODO need to leverage optional here
- //return deviceRef;
+ return Optional.empty();
}
ApplicationQueueMessage message = new ApplicationQueueMessage(appId, notification.getUuid(), deviceRef.getUuid(), notifierKey, notifierId);
@@ -157,15 +157,14 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
}
deviceCount.incrementAndGet();
- return message;
+ return Optional.of(message);
} catch (Exception deviceLoopException) {
logger.error("Failed to add device", deviceLoopException);
errorMessages.add("Failed to add device: " + deviceRef.getUuid() + ", error:" + deviceLoopException);
- //TODO need an optional here
- return new ApplicationQueueMessage(appId, notification.getUuid(), deviceRef.getUuid(), "test", "test");
+ return Optional.empty();
}
};
@@ -185,13 +184,21 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
applicationQueueMessages.forEach( message -> {
try {
-
- qm.sendMessage( message );
- queueMeter.mark();
+ if(message.isPresent()){
+ qm.sendMessage( message.get() );
+ queueMeter.mark();
+ }
} catch (IOException e) {
- logger.error("Unable to queue notification for notification UUID {} and device UUID {} ",
- message.getNotificationId(), message.getDeviceId());
+
+ if(message.isPresent()){
+ logger.error("Unable to queue notification for notification UUID {} and device UUID {} ",
+ message.get().getNotificationId(), message.get().getDeviceId());
+ }
+ else{
+ logger.error("Unable to queue notification as it's not present when trying to send to queue");
+ }
+
}
});
@@ -200,7 +207,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
})
.doOnError(throwable -> logger.error("Failed while trying to send notification", throwable));
- processMessagesObservable.toBlocking(); // let this run and block the async thread, messages are queued
+ processMessagesObservable.toBlocking().last(); // let this run and block the async thread, messages are queued
}
@@ -513,6 +520,8 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
List<EntityRef> devices = new ArrayList<>();
+ final int LIMIT = Query.MID_LIMIT;
+
try {
@@ -529,52 +538,48 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
initial = false;
- final List<EntityRef> mydevices = em.getCollection(ref, "devices", start, Query.DEFAULT_LIMIT,
+ final List<EntityRef> mydevices = em.getCollection(ref, "devices", start, LIMIT,
Query.Level.REFS, true).getRefs();
resultSize = mydevices.size();
+
if(mydevices.size() > 0){
start = mydevices.get(mydevices.size() - 1 ).getUuid();
}
-
devices.addAll( mydevices );
-
}
} else if ("group".equals(ref.getType())) {
- //devices = new ArrayList<>();
UUID start = null;
boolean initial = true;
int resultSize = 0;
- while( initial || resultSize >= Query.DEFAULT_LIMIT){
+ while( initial || resultSize >= LIMIT){
- initial = false;
- final List<EntityRef> myusers = em.getCollection(ref, "users", start,
- Query.DEFAULT_LIMIT, Query.Level.REFS, true).getRefs();
+ initial = false;
+ final List<EntityRef> myusers = em.getCollection(ref, "users", start,
+ LIMIT, Query.Level.REFS, true).getRefs();
- resultSize = myusers.size();
- if(myusers.size() > 0){
- start = myusers.get(myusers.size() - 1 ).getUuid();
- }
+ resultSize = myusers.size();
+ if(myusers.size() > 0){
+ start = myusers.get(myusers.size() - 1 ).getUuid();
+ }
- // don't allow a single user to have more than 100 devices?
- for (EntityRef user : myusers) {
- devices.addAll( em.getCollection(user, "devices", null, 100,
- Query.Level.REFS, true).getRefs() );
+ // don't allow a single user to have more than 100 devices?
+ for (EntityRef user : myusers) {
+ devices.addAll( em.getCollection(user, "devices", null, 100,
+ Query.Level.REFS, true).getRefs() );
- }
+ }
}
-
-
}
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c1375bf6/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
index 5895d38..9d95d87 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/queues/QueueListener.java
@@ -110,7 +110,7 @@ public abstract class QueueListener {
try {
sleepBetweenRuns = new Long(properties.getProperty("usergrid.queues.listener.sleep.between", ""+sleepBetweenRuns)).longValue();
sleepWhenNoneFound = new Long(properties.getProperty("usergrid.queues.listener.sleep.after", ""+DEFAULT_SLEEP)).longValue();
- batchSize = new Integer(properties.getProperty("usergrid.queues.listener.batchSize", (""+batchSize)));
+ batchSize = new Integer(properties.getProperty("usergrid.queues.listener.MAX_TAKE", (""+batchSize)));
consecutiveCallsToRemoveDevices = new Integer(properties.getProperty("usergrid.queues.inactive.interval", ""+200));
queueName = getQueueName();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c1375bf6/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
index dea4e49..3923827 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
@@ -129,7 +129,6 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
public void after() throws Exception {
if(listener != null) {
listener.stop();
- listener = null;
}
}
@@ -683,8 +682,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
final int NUM_DEVICES = 50;
// perform push //
- int oldBatchSize = listener.getBatchSize();
- listener.setBatchSize(10);
+ int oldBatchSize = QueueListener.MAX_TAKE;
app.clear();
app.put("name", UUID.randomUUID().toString());
@@ -724,7 +722,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
try {
notificationWaitForComplete(notification);
} finally {
- listener.setBatchSize( oldBatchSize);
+ //noop
}
// check receipts //
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c1375bf6/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 65cc54a..1c7915a 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -106,7 +106,6 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
public void after() {
if (listener != null) {
listener.stop();
- listener = null;
}
}
[08/50] usergrid git commit: Add flag to disable de-index of older
versions on entity updates.
Posted by sn...@apache.org.
Add flag to disable de-index of older versions on entity updates.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9f451306
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9f451306
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9f451306
Branch: refs/heads/master
Commit: 9f451306e74ff1816219ae4393ec9b5f9c5d1a41
Parents: ab028e9
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 5 17:33:44 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 5 17:33:44 2016 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/corepersistence/CpEntityManager.java | 4 +++-
.../org/apache/usergrid/corepersistence/EntityManagerFig.java | 4 ++++
.../apache/usergrid/corepersistence/StaleIndexCleanupTest.java | 3 ++-
3 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f451306/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index b29e6d3..bb9d61f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -593,7 +593,9 @@ public class CpEntityManager implements EntityManager {
// queue up an event to clean-up older versions than this one from the index
- indexService.queueDeIndexOldVersion( applicationScope, entityId );
+ if(entityManagerFig.getDeindexOnUpdate()) {
+ indexService.queueDeIndexOldVersion(applicationScope, entityId);
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f451306/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 45bebc6..4c50aee 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@ -37,4 +37,8 @@ public interface EntityManagerFig extends GuicyFig {
@Key( "usergrid.entityManager_sleep_ms" )
@Default( "100" )
int sleep();
+
+ @Key( "usergrid.entityManager.enable_deindex_on_update" )
+ @Default( "false" )
+ boolean getDeindexOnUpdate();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f451306/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 536f9f5..65c373f 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -70,6 +70,7 @@ import static org.junit.Assert.assertTrue;
* Test on read style clean-up of stale ElasticSearch indexes.
*/
@NotThreadSafe
+@Ignore
public class StaleIndexCleanupTest extends AbstractCoreIT {
private static final Logger logger = LoggerFactory.getLogger( StaleIndexCleanupTest.class );
public static final String EVENTS_DISABLED = "corepersistence.events.disabled";
@@ -133,7 +134,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
-
+
/**
* Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an
* entity is deleted its old indexes are cleared from ElasticSearch.
[41/50] usergrid git commit: Fix notification test ( receipts are
only created if debug = true ).
Posted by sn...@apache.org.
Fix notification test ( receipts are only created if debug = true ).
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/30266e13
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/30266e13
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/30266e13
Branch: refs/heads/master
Commit: 30266e13dd8055fd3c52689fa2b15020de98b015
Parents: 22a8d05
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Apr 15 07:31:44 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Apr 15 07:31:44 2016 +0200
----------------------------------------------------------------------
.../services/notifications/gcm/NotificationsServiceIT.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/30266e13/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 6ba237d..8782fe3 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -150,7 +150,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
payloads.put(notifier.getUuid().toString(), payload);
app.put("payloads", payloads);
app.put("queued", System.currentTimeMillis());
- app.put("debug", false);
+ app.put("debug", true);
app.put("expire", System.currentTimeMillis() + 300000); // add 5 minutes to current time
Entity e = app.testRequest(ServiceAction.POST, 1, "devices", device1.getUuid(), "notifications").getEntity();
[50/50] usergrid git commit: Merge branch 'release-2.1.1'
Posted by sn...@apache.org.
Merge branch 'release-2.1.1'
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9950af76
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9950af76
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9950af76
Branch: refs/heads/master
Commit: 9950af7684255e1949637ddbb8ca8effa74498a3
Parents: e0931be d069185
Author: Dave Johnson <sn...@apache.org>
Authored: Tue Apr 19 12:50:10 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue Apr 19 12:50:10 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 24 +
.../apache/usergrid/batch/job/OnlyOnceJob.java | 22 +-
.../corepersistence/ApplicationIdCache.java | 6 +-
.../corepersistence/ApplicationIdCacheImpl.java | 64 +-
.../usergrid/corepersistence/CoreModule.java | 6 +
.../corepersistence/CpEntityManager.java | 231 +++++-
.../corepersistence/CpEntityManagerFactory.java | 49 +-
.../corepersistence/CpManagerCache.java | 2 +-
.../corepersistence/CpRelationManager.java | 149 +++-
.../usergrid/corepersistence/CpSetup.java | 100 +--
.../corepersistence/EntityManagerFig.java | 4 +
.../usergrid/corepersistence/ManagerCache.java | 2 +-
.../asyncevents/AsyncEventService.java | 7 +
.../asyncevents/AsyncEventServiceImpl.java | 153 +++-
.../asyncevents/EventBuilder.java | 12 +-
.../asyncevents/EventBuilderImpl.java | 37 +-
.../asyncevents/model/AsyncEvent.java | 7 +-
.../model/DeIndexOldVersionsEvent.java | 50 ++
.../asyncevents/model/EdgeDeleteEvent.java | 4 +-
.../asyncevents/model/EdgeIndexEvent.java | 70 ++
.../model/ElasticsearchIndexEvent.java | 2 +-
.../asyncevents/model/EntityDeleteEvent.java | 3 +
.../asyncevents/model/EntityIndexEvent.java | 54 ++
.../model/InitializeApplicationIndexEvent.java | 2 +-
.../index/IndexProcessorFig.java | 13 +-
.../corepersistence/index/IndexSchemaCache.java | 50 ++
.../index/IndexSchemaCacheFactory.java | 44 ++
.../index/IndexSchemaCacheFig.java | 39 +
.../index/IndexSchemaCacheImpl.java | 119 +++
.../corepersistence/index/IndexServiceImpl.java | 98 ++-
.../index/ReIndexRequestBuilder.java | 7 +
.../index/ReIndexRequestBuilderImpl.java | 32 +
.../index/ReIndexServiceImpl.java | 49 +-
.../pipeline/builder/IdBuilder.java | 6 +
.../pipeline/read/FilterFactory.java | 8 +-
.../read/traverse/AbstractReadGraphFilter.java | 18 +-
.../pipeline/read/traverse/IdFilter.java | 52 ++
.../results/IdQueryExecutor.java | 66 ++
.../service/ApplicationServiceImpl.java | 37 +-
.../service/CollectionSearch.java | 9 +
.../service/CollectionService.java | 5 +
.../service/CollectionServiceImpl.java | 23 +
.../service/ServiceSchedulerFig.java | 2 +-
.../usergrid/persistence/EntityManager.java | 7 +
.../persistence/EntityManagerFactory.java | 6 +-
.../persistence/MultiQueryIterator.java | 2 +-
.../persistence/NotificationGraphIterator.java | 163 +++++
.../persistence/PagingResultsIterator.java | 25 +-
.../apache/usergrid/persistence/PathQuery.java | 67 +-
.../org/apache/usergrid/persistence/Query.java | 2 +
.../apache/usergrid/persistence/Results.java | 20 +
.../persistence/cassandra/ApplicationCF.java | 2 +-
.../persistence/cassandra/CassandraService.java | 2 -
.../usergrid/persistence/cassandra/Setup.java | 28 +-
.../persistence/entities/Notification.java | 106 ++-
.../corepersistence/StaleIndexCleanupTest.java | 184 +----
.../index/AsyncEventServiceImplTest.java | 23 +-
.../usergrid/corepersistence/index/RxTest.java | 129 ++++
.../usergrid/persistence/CoreSchemaManager.java | 14 +-
.../cassandra/EntityManagerFactoryImplIT.java | 8 +-
.../impl/ScopedCacheSerializationImpl.java | 9 +-
.../impl/EntityCollectionManagerImpl.java | 2 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 9 +-
.../serialization/SerializationFig.java | 3 +
.../core/astyanax/CassandraClusterImpl.java | 4 +-
.../persistence/core/astyanax/ColumnSearch.java | 7 +-
.../core/astyanax/MultiRowColumnIterator.java | 37 +-
.../astyanax/MultiRowShardColumnIterator.java | 462 ++++++++++++
.../core/consistency/TimeServiceImpl.java | 5 +-
.../core/executor/TaskExecutorFactory.java | 10 +-
.../persistence/core/shard/SmartShard.java | 61 ++
.../astyanax/MultiRowColumnIteratorTest.java | 14 +-
.../usergrid/persistence/graph/GraphFig.java | 17 +
.../graph/impl/GraphManagerImpl.java | 2 +-
.../impl/EdgeMetadataSerializationV2Impl.java | 4 +-
.../impl/shard/NodeShardCache.java | 7 +-
.../graph/serialization/impl/shard/Shard.java | 33 +-
.../impl/shard/impl/EdgeSearcher.java | 113 ++-
.../shard/impl/EdgeShardSerializationImpl.java | 56 +-
.../shard/impl/NodeShardAllocationImpl.java | 38 +-
.../impl/shard/impl/NodeShardCacheImpl.java | 29 +-
.../shard/impl/ShardEntryGroupIterator.java | 12 +
.../shard/impl/ShardGroupCompactionImpl.java | 179 +++--
.../impl/ShardedEdgeSerializationImpl.java | 51 +-
.../impl/shard/impl/ShardsColumnIterator.java | 64 +-
.../shard/impl/serialize/ShardSerializer.java | 86 +++
.../graph/GraphManagerShardConsistencyIT.java | 215 +++---
.../impl/shard/EdgeShardSerializationTest.java | 12 +-
.../impl/shard/NodeShardAllocationTest.java | 44 +-
.../impl/shard/ShardGroupCompactionTest.java | 4 +-
.../graph/src/test/resources/log4j.properties | 6 +
.../persistence/map/impl/MapManagerImpl.java | 6 +
stack/corepersistence/pom.xml | 2 +-
.../persistence/index/EntityIndexBatch.java | 13 +-
.../usergrid/persistence/index/IndexFig.java | 2 +-
.../index/impl/EntityToMapConverter.java | 93 ++-
.../index/impl/EsEntityIndexBatchImpl.java | 21 +-
.../index/impl/EsEntityIndexImpl.java | 25 +-
.../index/impl/FailureMonitorImpl.java | 2 +-
.../persistence/index/impl/IndexOperation.java | 9 +-
.../index/impl/IndexOperationMessage.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 2 +-
.../usergrid/persistence/queue/QueueFig.java | 4 +
.../apache/usergrid/mongo/BasicMongoTest.java | 8 +-
.../apache/usergrid/mongo/MongoQueryTest.java | 18 +-
stack/pom.xml | 2 +-
.../org/apache/usergrid/rest/ApiResponse.java | 7 +-
.../rest/applications/ApplicationResource.java | 2 +-
.../rest/applications/CollectionResource.java | 233 ++++++
.../rest/applications/ServiceResource.java | 140 +++-
.../notifiers/NotifierResource.java | 3 +-
.../notifiers/NotifiersResource.java | 3 +-
.../rest/applications/users/UserResource.java | 26 +-
.../rest/applications/users/UsersResource.java | 3 +-
.../IllegalArgumentExceptionMapper.java | 4 +-
.../ServiceResourceNotFoundExceptionMapper.java | 9 +-
.../organizations/OrganizationResource.java | 8 +-
.../security/SecuredResourceFilterFactory.java | 29 +-
.../OAuth2AccessTokenSecurityFilter.java | 3 +-
.../usergrid/rest/system/DatabaseResource.java | 4 +-
.../usergrid/rest/system/IndexResource.java | 6 +
.../rest/applications/ApplicationCreateIT.java | 3 +-
.../applications/ApplicationResourceIT.java | 2 +-
.../collection/CollectionsResourceIT.java | 716 ++++++++++++++++++-
.../collection/paging/PagingResourceIT.java | 70 +-
.../collection/users/PermissionsResourceIT.java | 6 +-
.../rest/test/resource/AbstractRestIT.java | 1 +
.../rest/test/resource/ClientSetup.java | 10 +
.../test/resource/endpoints/NamedResource.java | 22 +
.../resources/usergrid-custom-test.properties | 4 +
.../cassandra/ManagementServiceImpl.java | 91 ++-
.../usergrid/security/shiro/ShiroCache.java | 41 +-
.../security/shiro/ShiroCacheManager.java | 15 +-
.../shiro/credentials/AdminUserAccessToken.java | 3 +
.../ApplicationClientCredentials.java | 5 +
.../shiro/principals/AdminUserPrincipal.java | 3 +
.../principals/ApplicationGuestPrincipal.java | 4 +-
.../shiro/principals/ApplicationPrincipal.java | 4 +-
.../principals/ApplicationUserPrincipal.java | 3 +
.../shiro/principals/OrganizationPrincipal.java | 4 +-
.../security/shiro/utils/LocalShiroCache.java | 80 +++
.../usergrid/security/tokens/TokenService.java | 3 +
.../tokens/cassandra/TokenServiceImpl.java | 40 +-
.../services/AbstractCollectionService.java | 56 ++
.../services/AbstractConnectionsService.java | 13 +
.../usergrid/services/AbstractService.java | 14 +-
.../org/apache/usergrid/services/Service.java | 3 +
.../usergrid/services/ServiceManager.java | 1 -
.../usergrid/services/ServiceRequest.java | 5 +
.../applications/ApplicationsService.java | 14 +
.../usergrid/services/groups/GroupsService.java | 4 +
.../notifications/NotificationDeviceFilter.java | 45 ++
.../notifications/NotificationsService.java | 32 +-
.../services/notifications/QueueListener.java | 41 +-
.../services/notifications/TaskManager.java | 25 +-
.../services/notifications/gcm/GCMAdapter.java | 7 +-
.../impl/ApplicationQueueManagerImpl.java | 283 ++++++--
.../usergrid/services/queues/QueueListener.java | 2 +-
.../usergrid/services/roles/RolesService.java | 2 +
.../usergrid/services/users/UsersService.java | 2 +
.../services/users/roles/RolesService.java | 4 +
.../resources/usergrid-services-context.xml | 11 +-
.../migration/AppInfoMigrationPluginTest.java | 8 +-
.../apns/NotificationsServiceIT.java | 32 +-
.../gcm/NotificationsServiceIT.java | 58 +-
.../resources/usergrid-custom-test.properties | 1 +
.../usergrid/cassandra/SchemaManager.java | 3 -
.../usergrid/cassandra/FakeSchemaManager.java | 8 +-
.../test/notifications/notifications.js | 5 -
tests/performance/runLoadNestedEntities.sh | 98 +++
.../datagenerators/EntityDataGenerator.scala | 40 ++
.../org/apache/usergrid/enums/EntityType.scala | 3 +-
172 files changed, 5597 insertions(+), 1162 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9950af76/stack/corepersistence/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9950af76/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9950af76/stack/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9950af76/stack/services/src/main/java/org/apache/usergrid/security/tokens/cassandra/TokenServiceImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9950af76/tests/integration/test/notifications/notifications.js
----------------------------------------------------------------------
[35/50] usergrid git commit: Added test for google and apple to make
sure that the receipts flag is honored. Allow people to input strings and
boolean values to the receipts flag in order to save receipts. Added receipts
flag to notifications.
Posted by sn...@apache.org.
Added test for google and apple to make sure that the receipts flag is honored.
Allow people to input strings and boolean values to the receipts flag in order to save receipts.
Added receipts flag to notifications.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/491aa084
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/491aa084
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/491aa084
Branch: refs/heads/master
Commit: 491aa084f0c633f318dbe67cc566a6be81491b2e
Parents: 2e576f3
Author: George Reyes <gr...@apache.org>
Authored: Thu Apr 14 14:43:05 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Thu Apr 14 14:43:05 2016 -0700
----------------------------------------------------------------------
.../services/notifications/TaskManager.java | 18 ++++++------
.../impl/ApplicationQueueManagerImpl.java | 30 ++++++++++++++++++--
.../apns/NotificationsServiceIT.java | 26 +++++++++++++++++
.../gcm/NotificationsServiceIT.java | 27 +++++++++++++++++-
4 files changed, 90 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/491aa084/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
index 954724f..ce2b82c 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskManager.java
@@ -104,14 +104,16 @@ public class TaskManager {
}
failures.incrementAndGet();
- if (receipt.getUuid() != null) {
- successes.decrementAndGet();
- }
- receipt.setErrorCode(code);
- receipt.setErrorMessage(message);
- this.saveReceipt(notification, new SimpleEntityRef(Device.ENTITY_TYPE, deviceUUID), receipt,true);
- if (logger.isDebugEnabled()) {
- logger.debug("notification {} receipt saved for device {}", notification.getUuid(), deviceUUID);
+ if(receipt!=null) {
+ if ( receipt.getUuid() != null ) {
+ successes.decrementAndGet();
+ }
+ receipt.setErrorCode( code );
+ receipt.setErrorMessage( message );
+ this.saveReceipt( notification, new SimpleEntityRef( Device.ENTITY_TYPE, deviceUUID ), receipt, true );
+ if ( logger.isDebugEnabled() ) {
+ logger.debug( "notification {} receipt saved for device {}", notification.getUuid(), deviceUUID );
+ }
}
} finally {
completed(notifier, deviceUUID);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/491aa084/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 0b0432d..8b908ee 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -329,8 +329,34 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
String notifierName = message.getNotifierKey().toLowerCase();
ProviderAdapter providerAdapter = notifierMap.get(notifierName.toLowerCase());
Object payload = translatedPayloads.get(notifierName);
- Receipt receipt = new Receipt(notification.getUuid(), message.getNotifierId(), payload, deviceUUID);
- TaskTracker tracker = new TaskTracker(providerAdapter.getNotifier(), taskManager, receipt, deviceUUID);
+ Map dynamicNotificationProperties =notification.getDynamicProperties();
+ //do additional error checking to make sure other values will work.
+ Boolean receiptsField = true;
+
+ Object typelessReceiptsField = dynamicNotificationProperties.getOrDefault( "receipts",true );
+ if(typelessReceiptsField instanceof Boolean) {
+ receiptsField = ( boolean ) typelessReceiptsField;
+ }
+ else if(typelessReceiptsField instanceof String){
+ String booleanString = ( String ) typelessReceiptsField;
+ if(booleanString.toLowerCase().equals( "false" )){
+ receiptsField=false;
+ }
+ }
+
+ TaskTracker tracker = null;
+ if(receiptsField==false){
+// Receipt receipt =
+// new Receipt( notification.getUuid(), message.getNotifierId(), payload, deviceUUID );
+ tracker =
+ new TaskTracker( providerAdapter.getNotifier(), taskManager, null, deviceUUID );
+ }
+ else {
+ Receipt receipt =
+ new Receipt( notification.getUuid(), message.getNotifierId(), payload, deviceUUID );
+ tracker =
+ new TaskTracker( providerAdapter.getNotifier(), taskManager, receipt, deviceUUID );
+ }
if (!isOkToSend(notification)) {
tracker.failed(0, "Notification is duplicate/expired/cancelled.");
} else {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/491aa084/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
index 3923827..0ef5fe4 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
@@ -163,6 +163,32 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
}
@Test
+ public void singlePushNotificationNoReceipts() throws Exception {
+
+ app.clear();
+ String payload = "Hello, World!";
+ Map<String, String> payloads = new HashMap<String, String>(1);
+ payloads.put(notifier.getUuid().toString(), payload);
+ app.put("payloads", payloads);
+ app.put("queued", System.currentTimeMillis());
+ app.put("debug", true);
+ app.put("receipts",false );
+ app.put("expire", System.currentTimeMillis() + 300000); // add 5 minutes to current time
+
+ Entity e = app.testRequest(ServiceAction.POST, 1, "devices", device1.getUuid(), "notifications").getEntity();
+ app.testRequest(ServiceAction.GET, 1, "notifications", e.getUuid());
+
+ Notification notification = app.getEntityManager().get(e.getUuid(), Notification.class);
+ assertEquals(
+ notification.getPayloads().get(notifier.getUuid().toString()),
+ payload);
+
+ // perform push //
+ notification = notificationWaitForComplete(notification);
+ checkReceipts(notification, 0);
+ }
+
+ @Test
public void pushWithNoValidDevicesShouldComplete() throws Exception {
// create unrelated notifier
http://git-wip-us.apache.org/repos/asf/usergrid/blob/491aa084/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 1c7915a..13b2887 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -150,7 +150,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
payloads.put(notifier.getUuid().toString(), payload);
app.put("payloads", payloads);
app.put("queued", System.currentTimeMillis());
- app.put("debug", true);
+ app.put("debug", false);
app.put("expire", System.currentTimeMillis() + 300000); // add 5 minutes to current time
Entity e = app.testRequest(ServiceAction.POST, 1, "devices", device1.getUuid(), "notifications").getEntity();
@@ -166,6 +166,31 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
checkReceipts(notification, 1);
}
+ @Test
+ public void singlePushNotificationNoReceipts() throws Exception {
+
+ app.clear();
+ String payload = "Hello, World!";
+ Map<String, String> payloads = new HashMap<String, String>(1);
+ payloads.put(notifier.getUuid().toString(), payload);
+ app.put("payloads", payloads);
+ app.put("queued", System.currentTimeMillis());
+ app.put("debug", true);
+ app.put("receipts",false );
+ app.put("expire", System.currentTimeMillis() + 300000); // add 5 minutes to current time
+
+ Entity e = app.testRequest(ServiceAction.POST, 1, "devices", device1.getUuid(), "notifications").getEntity();
+ app.testRequest(ServiceAction.GET, 1, "notifications", e.getUuid());
+
+ Notification notification = app.getEntityManager().get(e.getUuid(), Notification.class);
+ assertEquals(
+ notification.getPayloads().get(notifier.getUuid().toString()),
+ payload);
+
+ // perform push //
+ notification = notificationWaitForComplete(notification);
+ checkReceipts(notification, 0);
+ }
@Test
public void singlePushNotificationHighPriority() throws Exception {
[10/50] usergrid git commit: Add OnlyOnceJob fix to 9f45130
Posted by sn...@apache.org.
Add OnlyOnceJob fix to 9f45130
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/55271ce9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/55271ce9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/55271ce9
Branch: refs/heads/master
Commit: 55271ce9347bd4f35224a2c31494b23fa371ae67
Parents: 9f45130
Author: Mike Dunker <md...@apigee.com>
Authored: Thu Apr 7 15:33:43 2016 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Thu Apr 7 15:33:43 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/batch/job/OnlyOnceJob.java | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/55271ce9/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
index f857d05..afbddc3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
+++ b/stack/core/src/main/java/org/apache/usergrid/batch/job/OnlyOnceJob.java
@@ -19,7 +19,7 @@ package org.apache.usergrid.batch.job;
import java.util.concurrent.TimeUnit;
-import org.elasticsearch.common.inject.Inject;
+import com.google.inject.Injector;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.apache.usergrid.batch.Job;
@@ -28,6 +28,8 @@ import org.apache.usergrid.locking.Lock;
import org.apache.usergrid.locking.LockManager;
import org.apache.usergrid.persistence.EntityManagerFactory;
+import javax.annotation.PostConstruct;
+
/**
* Simple abstract job class that performs additional locking to ensure that the job is only executing once. This can be
@@ -38,12 +40,14 @@ import org.apache.usergrid.persistence.EntityManagerFactory;
@Component("OnlyOnceJob")
public abstract class OnlyOnceJob implements Job {
- @Inject
private LockManager lockManager;
@Autowired
private EntityManagerFactory emf;
+ @Autowired
+ private Injector injector;
+
/**
*
@@ -51,6 +55,10 @@ public abstract class OnlyOnceJob implements Job {
public OnlyOnceJob() {
}
+ @PostConstruct
+ public void initLockManager() throws Exception {
+ this.lockManager = injector.getInstance(LockManager.class);
+ }
/*
* (non-Javadoc)
[07/50] usergrid git commit: Revert "Reduce number of old ES indexes
that are being deleted at a time."
Posted by sn...@apache.org.
Revert "Reduce number of old ES indexes that are being deleted at a time."
This reverts commit dcd8c39f396bffa79395ef504c9640eb492dde8f.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ab028e96
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ab028e96
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ab028e96
Branch: refs/heads/master
Commit: ab028e96a7f44ccf35a4474a21143c46a25556a5
Parents: 90217b5
Author: Michael Russo <mr...@apigee.com>
Authored: Tue Apr 5 13:47:39 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Tue Apr 5 13:47:39 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/persistence/index/IndexFig.java | 6 ------
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 2 +-
2 files changed, 1 insertion(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ab028e96/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 447169e..e81219a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -62,8 +62,6 @@ public interface IndexFig extends GuicyFig {
String ELASTICSEARCH_VERSION_QUERY_LIMIT = "elasticsearch.version_query_limit";
- String ELASTICSEARCH_OLD_VERSION_QUERY_LIMIT = "elasticsearch.old_version_query_limit";
-
/**
* Comma-separated list of Elasticsearch hosts.
@@ -207,8 +205,4 @@ public interface IndexFig extends GuicyFig {
@Default("1000")
@Key( ELASTICSEARCH_VERSION_QUERY_LIMIT )
int getVersionQueryLimit();
-
- @Default("10")
- @Key( ELASTICSEARCH_OLD_VERSION_QUERY_LIMIT )
- int getOldVersionQueryLimit();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ab028e96/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index f754cc2..f5da29a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -545,7 +545,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
final long markedTimestamp = markedVersion.timestamp();
// never let the limit be less than 2 as there are potential indefinite paging issues
- final int searchLimit = Math.max(2, indexFig.getOldVersionQueryLimit());
+ final int searchLimit = Math.max(2, indexFig.getVersionQueryLimit());
// this query will find the document for the entity itself
final QueryBuilder entityQuery = QueryBuilders
[39/50] usergrid git commit: Update test with new property name.
Posted by sn...@apache.org.
Update test with new property name.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2e344cb7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2e344cb7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2e344cb7
Branch: refs/heads/master
Commit: 2e344cb731ecc78f06034d6e001c04d1db2bcf60
Parents: 8dce5ed
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Apr 15 01:02:37 2016 +0200
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Apr 15 01:02:37 2016 +0200
----------------------------------------------------------------------
.../services/notifications/apns/NotificationsServiceIT.java | 2 +-
.../services/notifications/gcm/NotificationsServiceIT.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e344cb7/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
index 0ef5fe4..a3ac99f 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
@@ -172,7 +172,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
app.put("payloads", payloads);
app.put("queued", System.currentTimeMillis());
app.put("debug", true);
- app.put("receipts",false );
+ app.put("saveReceipts",false );
app.put("expire", System.currentTimeMillis() + 300000); // add 5 minutes to current time
Entity e = app.testRequest(ServiceAction.POST, 1, "devices", device1.getUuid(), "notifications").getEntity();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e344cb7/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 13b2887..6ba237d 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -176,7 +176,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
app.put("payloads", payloads);
app.put("queued", System.currentTimeMillis());
app.put("debug", true);
- app.put("receipts",false );
+ app.put("saveReceipts",false );
app.put("expire", System.currentTimeMillis() + 300000); // add 5 minutes to current time
Entity e = app.testRequest(ServiceAction.POST, 1, "devices", device1.getUuid(), "notifications").getEntity();
[44/50] usergrid git commit: Update notification processing to allow
more parallel work.
Posted by sn...@apache.org.
Update notification processing to allow more parallel work.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8cf78252
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8cf78252
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8cf78252
Branch: refs/heads/master
Commit: 8cf782527f705109fdbedd8d8767e6074e42796a
Parents: 32ab5da
Author: Michael Russo <mr...@apigee.com>
Authored: Sat Apr 16 14:40:20 2016 +0100
Committer: Michael Russo <mr...@apigee.com>
Committed: Sat Apr 16 14:40:20 2016 +0100
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 47 ++++--
.../IllegalArgumentExceptionMapper.java | 4 +-
.../notifications/NotificationsService.java | 27 ++--
.../impl/ApplicationQueueManagerImpl.java | 145 ++++++++++---------
4 files changed, 128 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8cf78252/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index b2330f3..750cf7b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -34,7 +34,9 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
+import java.util.stream.Collectors;
+import org.apache.usergrid.persistence.collection.EntitySet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -2511,23 +2513,42 @@ public class CpEntityManager implements EntityManager {
@Override
public Results getEntities( List<UUID> ids, String type ) {
- ArrayList<Entity> entities = new ArrayList<Entity>();
- for ( UUID uuid : ids ) {
- EntityRef ref = new SimpleEntityRef( type, uuid );
- Entity entity = null;
- try {
- entity = get( ref );
- }
- catch ( Exception ex ) {
- logger.warn( "Entity {}/{} not found", uuid, type );
- }
- if ( entity != null ) {
- entities.add( entity );
- }
+ List<Id> entityIds = new ArrayList<>();
+
+ for( UUID uuid : ids){
+
+ entityIds.add(new SimpleId( uuid, type ));
+
}
+ // leverage ecm.load so it's a batch fetch of all entities from Cassandra
+ EntitySet entitySet = ecm.load( entityIds ).toBlocking().last();
+
+ List<Entity> entities = entitySet.getEntities().stream().map( mvccEntity -> {
+
+ if( mvccEntity.getEntity().isPresent() ){
+
+ org.apache.usergrid.persistence.model.entity.Entity cpEntity = mvccEntity.getEntity().get();
+
+ Class clazz = Schema.getDefaultSchema().getEntityClass( mvccEntity.getId().getType() );
+
+ Entity entity = EntityFactory.newEntity( mvccEntity.getId().getUuid(), mvccEntity.getId().getType(), clazz );
+ entity.setProperties( cpEntity );
+
+ return entity;
+
+ }else{
+
+ logger.warn("Tried fetching entity with id: {} and type: but was not found",
+ mvccEntity.getId().getUuid(), mvccEntity.getId().getType() );
+
+ return null;
+ }
+ }).collect(Collectors.toList());
+
+
return Results.fromEntities( entities );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8cf78252/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/IllegalArgumentExceptionMapper.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/IllegalArgumentExceptionMapper.java b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/IllegalArgumentExceptionMapper.java
index ff7b656..e6243e9 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/IllegalArgumentExceptionMapper.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/exceptions/IllegalArgumentExceptionMapper.java
@@ -34,7 +34,9 @@ public class IllegalArgumentExceptionMapper extends AbstractExceptionMapper<Ille
@Override
public Response toResponse( IllegalArgumentException e ) {
- logger.error( "Illegal argument was passed, returning bad request to user", e );
+ if(logger.isTraceEnabled()) {
+ logger.trace("Illegal argument was passed, returning bad request to user", e.getMessage());
+ }
return toResponse( BAD_REQUEST, e );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8cf78252/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
index 50eb883..f4fdb65 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.services.notifications;
import java.util.*;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.services.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,15 +40,6 @@ import org.apache.usergrid.persistence.queue.QueueManager;
import org.apache.usergrid.persistence.queue.QueueManagerFactory;
import org.apache.usergrid.persistence.queue.QueueScope;
import org.apache.usergrid.persistence.queue.impl.QueueScopeImpl;
-import org.apache.usergrid.services.AbstractCollectionService;
-import org.apache.usergrid.services.ServiceAction;
-import org.apache.usergrid.services.ServiceContext;
-import org.apache.usergrid.services.ServiceInfo;
-import org.apache.usergrid.services.ServiceManagerFactory;
-import org.apache.usergrid.services.ServiceParameter;
-import org.apache.usergrid.services.ServicePayload;
-import org.apache.usergrid.services.ServiceRequest;
-import org.apache.usergrid.services.ServiceResults;
import org.apache.usergrid.services.exceptions.ForbiddenServiceOperationException;
import org.apache.usergrid.services.notifications.impl.ApplicationQueueManagerImpl;
@@ -85,7 +77,6 @@ public class NotificationsService extends AbstractCollectionService {
private ServiceManagerFactory smf;
private EntityManagerFactory emf;
private QueueManagerFactory queueManagerFactory;
- private EntityCollectionManagerFactory ecmf;
public NotificationsService() {
if (logger.isTraceEnabled()) {
@@ -139,10 +130,20 @@ public class NotificationsService extends AbstractCollectionService {
Timer.Context timer = postTimer.time();
postMeter.mark();
try {
+
validate(null, context.getPayload());
- Notification.PathTokens pathTokens = getPathTokens(context.getRequest().getOriginalParameters());
- // default saving of receipts
+ // perform some input validates on useGraph payload property vs. ql= path query
+ final List<ServiceParameter> parameters = context.getRequest().getOriginalParameters();
+ for (ServiceParameter parameter : parameters){
+ if( parameter instanceof ServiceParameter.QueryParameter && context.getProperties().get("useGraph").equals(true)){
+ throw new IllegalArgumentException("Query ql parameter cannot be used with useGraph:true property value");
+ }
+ }
+
+ Notification.PathTokens pathTokens = getPathTokens(parameters);
+
+ // set defaults
context.getProperties().put("filters", context.getProperties().getOrDefault("filters", new HashMap<>()));
context.getProperties().put("useGraph", context.getProperties().getOrDefault("useGraph", false));
context.getProperties().put("saveReceipts", context.getProperties().getOrDefault("saveReceipts", true));
@@ -175,7 +176,7 @@ public class NotificationsService extends AbstractCollectionService {
// future: somehow return 202?
return results;
}catch (Exception e){
- logger.error("serialization failed",e);
+ logger.error(e.getMessage());
throw e;
}finally {
timer.stop();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8cf78252/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index 487ea1f..5ce1b93 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -58,7 +58,9 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
HashMap<Object, ProviderAdapter> notifierHashMap; // only retrieve notifiers once
- public ApplicationQueueManagerImpl(JobScheduler jobScheduler, EntityManager entityManager, QueueManager queueManager, MetricsFactory metricsFactory, Properties properties) {
+ public ApplicationQueueManagerImpl( JobScheduler jobScheduler, EntityManager entityManager,
+ QueueManager queueManager, MetricsFactory metricsFactory,
+ Properties properties) {
this.em = entityManager;
this.qm = queueManager;
this.jobScheduler = jobScheduler;
@@ -116,21 +118,24 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
iterator = pathQuery.iterator(em);
}
- //if there are more pages (defined by PAGE_SIZE) you probably want this to be async, also if this is already a job then don't reschedule
- if (iterator instanceof ResultsIterator && ((ResultsIterator) iterator).hasPages() && jobExecution == null) {
- if(logger.isTraceEnabled()){
- logger.trace("Scheduling notification job as it has multiple pages of devices.");
- }
- jobScheduler.scheduleQueueJob(notification, true);
- em.update(notification);
- return;
- }
+// //if there are more pages (defined by PAGE_SIZE) you probably want this to be async, also if this is already a job then don't reschedule
+// if (iterator instanceof ResultsIterator && ((ResultsIterator) iterator).hasPages() && jobExecution == null) {
+// if(logger.isTraceEnabled()){
+// logger.trace("Scheduling notification job as it has multiple pages of devices.");
+// }
+// jobScheduler.scheduleQueueJob(notification, true);
+// em.update(notification);
+// return;
+// }
final UUID appId = em.getApplication().getUuid();
final Map<String, Object> payloads = notification.getPayloads();
final Func1<EntityRef, Optional<ApplicationQueueMessage>> sendMessageFunction = deviceRef -> {
+
try {
+ //logger.info("Preparing notification queue message for device: {}", deviceRef.getUuid());
+
long now = System.currentTimeMillis();
String notifierId = null;
@@ -163,6 +168,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
notification.setQueued(System.currentTimeMillis());
}
+
deviceCount.incrementAndGet();
return Optional.of(message);
@@ -190,90 +196,95 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
return Observable.from(getDevices(entity));
})
- .distinct(ref -> ref.getUuid())
- .flatMap( entityRef -> {
+ .distinct(ref -> ref.getUuid() )
+ .map( entityRef -> entityRef.getUuid() )
+ .buffer(10)
+ .flatMap( uuids -> {
- return Observable.just(entityRef).flatMap( ref -> {
+ if(logger.isTraceEnabled()) {
+ logger.trace("Processing batch of {} device(s)", uuids.size());
+ }
- if(logger.isTraceEnabled()){
- logger.trace("Loading device: {}", ref.getUuid());
- }
- try {
- return Observable.just(em.get(ref, Device.class));
- }
- catch (Exception e){
-
- return Observable.empty();
+ return Observable.from(em.getEntities(uuids, "device"))
+ .filter( device -> {
+ if(logger.isTraceEnabled()) {
+ logger.trace("Filtering device: {}", device.getUuid());
}
- }).subscribeOn(Schedulers.io());
+ if(notification.getUseGraph() && filters.size() > 0 ) {
- }, 50)
- .filter( device -> {
+ for (Map.Entry<String, Object> entry : filters.entrySet()) {
- if(logger.isTraceEnabled()) {
- logger.trace("Filtering device: {}", device.getUuid());
- }
+ if ((device.getDynamicProperties().get(entry.getKey()) != null &&
+ device.getDynamicProperties().get(entry.getKey()).equals(entry.getValue())) ||
+ (device.getProperties().get(entry.getKey()) != null &&
+ device.getProperties().get(entry.getKey()).equals(entry.getValue()))
- if(notification.getUseGraph() && filters.size() > 0 ) {
+ ) {
- for (Map.Entry<String, Object> entry : filters.entrySet()) {
- if ((device.getDynamicProperties().get(entry.getKey()) != null &&
- device.getDynamicProperties().get(entry.getKey()).equals(entry.getValue())) ||
+ return true;
+ }
- (device.getProperties().get(entry.getKey()) != null &&
- device.getProperties().get(entry.getKey()).equals(entry.getValue()))
+ }
- ) {
+ if(logger.isTraceEnabled()) {
+ logger.trace("Push notification filter matched for notification {}, so removing from notification",
+ device.getUuid(), notification.getUuid());
+ }
+ return false;
- return true;
}
- }
+ return true;
- if(logger.isTraceEnabled()) {
- logger.trace("Push notification filter matched for notification {}, so removing from notification",
- device.getUuid(), notification.getUuid());
- }
- return false;
-
-
- }
+ })
+ .map(sendMessageFunction)
+ .doOnNext( message -> {
+ try {
- return true;
+ if(message.isPresent()){
- })
- .map(sendMessageFunction)
- .doOnNext( message -> {
- try {
+ if(logger.isTraceEnabled()) {
+ logger.trace("Queueing notification message for device: {}", message.get().getDeviceId());
+ }
+ qm.sendMessage( message.get() );
+ queueMeter.mark();
+ }
- if(message.isPresent()){
+ } catch (IOException e) {
- if(logger.isTraceEnabled()) {
- logger.trace("Queueing notification message for device: {}", message.get().getDeviceId());
+ if(message.isPresent()){
+ logger.error("Unable to queue notification for notification UUID {} and device UUID {} ",
+ message.get().getNotificationId(), message.get().getDeviceId());
+ }
+ else{
+ logger.error("Unable to queue notification as it's not present when trying to send to queue");
}
- qm.sendMessage( message.get() );
- queueMeter.mark();
+
}
- } catch (IOException e) {
- if(message.isPresent()){
- logger.error("Unable to queue notification for notification UUID {} and device UUID {} ",
- message.get().getNotificationId(), message.get().getDeviceId());
- }
- else{
- logger.error("Unable to queue notification as it's not present when trying to send to queue");
- }
+ }).subscribeOn(Schedulers.io());
+ }, 10)
- }
+ .doOnError(throwable -> {
+ logger.error("Error while processing devices for notification : {}", notification.getUuid());
+ notification.setProcessingFinished(-1L);
+ notification.setDeviceProcessedCount(deviceCount.get());
+ logger.warn("Partial notification. Only {} devices processed for notification {}",
+ deviceCount.get(), notification.getUuid());
+ try {
+ em.update(notification);
+ }catch (Exception e){
+ logger.error("Error updating negative processing status when processing failed.");
+ }
})
.doOnCompleted( () -> {
@@ -282,16 +293,14 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
notification.setProcessingFinished(System.currentTimeMillis());
notification.setDeviceProcessedCount(deviceCount.get());
em.update(notification);
- logger.info("{} devices processed for notification {}", deviceCount.get(), notification.getUuid());
+ logger.info("{} device(s) processed for notification {}", deviceCount.get(), notification.getUuid());
} catch (Exception e) {
logger.error("Unable to set processing finished timestamp for notification");
}
- })
- .doOnError(throwable -> logger.error("Failed while trying to send notification", throwable));
+ });
- //TODO verify error handling here
processMessagesObservable.subscribeOn(Schedulers.io()).subscribe(); // fire the queuing into the background
}