You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/01 16:51:37 UTC
[01/35] git commit: pick random queue between 0 and queue size
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-rebuildable-index 1aa04a71d -> 3df5d4d25
pick random queue between 0 and queue size
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f2a93d7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f2a93d7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f2a93d7f
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: f2a93d7f3283a148b2091821f2644506313def8f
Parents: 4cba3b8
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Sep 26 11:02:25 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Sep 26 11:02:25 2014 -0600
----------------------------------------------------------------------
.../notifications/ApplicationQueueManager.java | 27 +++++++++++---------
.../services/notifications/QueueListener.java | 15 +++++------
.../services/notifications/QueueManager.java | 4 ---
.../notifications/SingleQueueTaskManager.java | 8 +++---
4 files changed, 26 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f2a93d7f/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
index d110b95..198d41a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
@@ -18,15 +18,9 @@ package org.apache.usergrid.services.notifications;
import com.clearspring.analytics.hash.MurmurHash;
import com.clearspring.analytics.stream.frequency.CountMinSketch;
-import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.metrics.MetricsFactory;
-import org.apache.usergrid.mq.QueueQuery;
-import org.apache.usergrid.mq.QueueResults;
import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.entities.Device;
import org.apache.usergrid.persistence.entities.Notification;
@@ -66,7 +60,7 @@ public class ApplicationQueueManager implements QueueManager {
private final org.apache.usergrid.mq.QueueManager qm;
private final JobScheduler jobScheduler;
private final MetricsFactory metricsFactory;
- private final String queueName;
+ private final String[] queueNames;
HashMap<Object, Notifier> notifierHashMap; // only retrieve notifiers once
@@ -87,7 +81,7 @@ public class ApplicationQueueManager implements QueueManager {
this.qm = queueManager;
this.jobScheduler = jobScheduler;
this.metricsFactory = metricsFactory;
- this.queueName = properties.getProperty(DEFAULT_QUEUE_PROPERTY, DEFAULT_QUEUE_NAME);
+ this.queueNames = getQueueNames(properties);
}
@@ -115,6 +109,7 @@ public class ApplicationQueueManager implements QueueManager {
final ConcurrentLinkedQueue<String> errorMessages = new ConcurrentLinkedQueue<String>(); //build up list of issues
final HashMap<Object,Notifier> notifierMap = getNotifierMap();
+ final String queueName = getRandomQueue(queueNames);
//get devices in querystring, and make sure you have access
if (pathQuery != null) {
@@ -235,7 +230,7 @@ public class ApplicationQueueManager implements QueueManager {
//do i have devices, and have i already started batching.
if (deviceCount.get() <= 0) {
- SingleQueueTaskManager taskManager = new SingleQueueTaskManager(em, qm, this, notification);
+ SingleQueueTaskManager taskManager = new SingleQueueTaskManager(em, qm, this, notification,queueName);
//if i'm in a test value will be false, do not mark finished for test orchestration, not ideal need real tests
taskManager.finishedBatch();
}
@@ -288,7 +283,7 @@ public class ApplicationQueueManager implements QueueManager {
* @param messages
* @throws Exception
*/
- public Observable sendBatchToProviders( final List<ApplicationQueueMessage> messages) {
+ public Observable sendBatchToProviders( final List<ApplicationQueueMessage> messages, final String queuePath) {
LOG.info("sending batch of {} notifications.", messages.size());
final Meter sendMeter = metricsFactory.getMeter(NotificationsService.class, "send");
@@ -313,7 +308,7 @@ public class ApplicationQueueManager implements QueueManager {
SingleQueueTaskManager taskManager;
taskManager = taskMap.get(message.getNotificationId());
if (taskManager == null) {
- taskManager = new SingleQueueTaskManager(em, qm, proxy, notification);
+ taskManager = new SingleQueueTaskManager(em, qm, proxy, notification,queuePath);
taskMap.putIfAbsent(message.getNotificationId(), taskManager);
taskManager = taskMap.get(message.getNotificationId());
}
@@ -427,6 +422,15 @@ public class ApplicationQueueManager implements QueueManager {
return translatedPayloads;
}
+ public static String[] getQueueNames(Properties properties) {
+ String[] names = properties.getProperty(ApplicationQueueManager.DEFAULT_QUEUE_PROPERTY,ApplicationQueueManager.DEFAULT_QUEUE_NAME).split(";");
+ return names;
+ }
+ public static String getRandomQueue(String[] queueNames) {
+ int size = queueNames.length;
+ Random random = new Random();
+ return queueNames[random.nextInt(size)];
+ }
private static final class IteratorObservable<T> implements rx.Observable.OnSubscribe<T> {
private final Iterator<T> input;
@@ -571,6 +575,5 @@ public class ApplicationQueueManager implements QueueManager {
}
}
- public String getQueuePath(){return queueName;}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f2a93d7f/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index 6e9a7ef..1d4c5cb 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -35,8 +35,6 @@ import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
public class QueueListener {
- public static int MAX_CONSECUTIVE_FAILS = 10000;
-
public static final long MESSAGE_TRANSACTION_TIMEOUT = 60 * 5 * 1000;
private static final Logger LOG = LoggerFactory.getLogger(QueueListener.class);
@@ -66,7 +64,7 @@ public class QueueListener {
public static final String MAX_THREADS = "1";
private Integer batchSize = 1000;
- private String queueName;
+ private String[] queueNames;
public QueueListener() {
pool = Executors.newFixedThreadPool(1);
@@ -95,7 +93,7 @@ public class QueueListener {
sleepBetweenRuns = new Long(properties.getProperty("usergrid.notifications.listener.sleep.between", "0")).longValue();
sleepWhenNoneFound = new Long(properties.getProperty("usergrid.notifications.listener.sleep.after", "5000")).longValue();
batchSize = new Integer(properties.getProperty("usergrid.notifications.listener.batchSize", (""+batchSize)));
- queueName = properties.getProperty(ApplicationQueueManager.DEFAULT_QUEUE_PROPERTY,ApplicationQueueManager.DEFAULT_QUEUE_NAME);
+ queueNames = ApplicationQueueManager.getQueueNames(properties);
int maxThreads = new Integer(properties.getProperty("usergrid.notifications.listener.maxThreads", MAX_THREADS));
futures = new ArrayList<Future>(maxThreads);
@@ -133,7 +131,8 @@ public class QueueListener {
// run until there are no more active jobs
while ( true ) {
try {
- QueueResults results = getDeliveryBatch(queueManager);
+ String queueName = ApplicationQueueManager.getRandomQueue(queueNames);
+ QueueResults results = getDeliveryBatch(queueManager,queueName);
LOG.info("QueueListener: retrieved batch of {} messages", results.size());
List<Message> messages = results.getMessages();
@@ -167,7 +166,7 @@ public class QueueListener {
);
LOG.info("QueueListener: send batch for app {} of {} messages", entry.getKey(), entry.getValue().size());
- Observable current = manager.sendBatchToProviders(entry.getValue());
+ Observable current = manager.sendBatchToProviders(entry.getValue(),queueName);
if(merge == null)
merge = current;
else {
@@ -211,11 +210,11 @@ public class QueueListener {
}
}
- private QueueResults getDeliveryBatch(org.apache.usergrid.mq.QueueManager queueManager) throws Exception {
+ private QueueResults getDeliveryBatch(org.apache.usergrid.mq.QueueManager queueManager,String queuePath) throws Exception {
QueueQuery qq = new QueueQuery();
qq.setLimit(this.getBatchSize());
qq.setTimeout(MESSAGE_TRANSACTION_TIMEOUT);
- QueueResults results = queueManager.getFromQueue(queueName, qq);
+ QueueResults results = queueManager.getFromQueue(queuePath, qq);
LOG.debug("got batch of {} devices", results.size());
return results;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f2a93d7f/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueManager.java
index f92d463..0024417 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueManager.java
@@ -26,10 +26,6 @@ import java.util.Set;
*/
public interface QueueManager {
- public HashMap<Object,Notifier> getNotifierMap();
-
public void asyncCheckForInactiveDevices(Set<Notifier> notifiers) throws Exception ;
- public String getQueuePath();
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f2a93d7f/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java
index 8b4866f..f87f497 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java
@@ -34,8 +34,8 @@ public class SingleQueueTaskManager implements NotificationsTaskManager {
private static final Logger LOG = LoggerFactory
.getLogger(SingleQueueTaskManager.class);
- private final String path;
private final QueueManager proxy;
+ private final String queuePath;
private Notification notification;
private AtomicLong successes = new AtomicLong();
@@ -45,14 +45,14 @@ public class SingleQueueTaskManager implements NotificationsTaskManager {
private ConcurrentHashMap<UUID, ApplicationQueueMessage> messageMap;
private boolean hasFinished;
- public SingleQueueTaskManager(EntityManager em, org.apache.usergrid.mq.QueueManager qm, QueueManager proxy, Notification notification) {
+ public SingleQueueTaskManager(EntityManager em, org.apache.usergrid.mq.QueueManager qm, QueueManager proxy, Notification notification,String queuePath) {
this.em = em;
this.qm = qm;
- this.path = proxy.getQueuePath();
this.notification = notification;
this.proxy = proxy;
this.messageMap = new ConcurrentHashMap<UUID, ApplicationQueueMessage>();
hasFinished = false;
+ this.queuePath = queuePath;
}
public void addMessage(UUID deviceId, ApplicationQueueMessage message) {
@@ -72,7 +72,7 @@ public class SingleQueueTaskManager implements NotificationsTaskManager {
}
LOG.debug("notification {} removing device {} from remaining", notification.getUuid(), deviceUUID);
- qm.commitTransaction(path, messageMap.get(deviceUUID).getTransaction(), null);
+ qm.commitTransaction(queuePath, messageMap.get(deviceUUID).getTransaction(), null);
if (newProviderId != null) {
LOG.debug("notification {} replacing device {} notifierId", notification.getUuid(), deviceUUID);
replaceProviderId(deviceRef, notifier, newProviderId);
[32/35] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f6a79d05
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f6a79d05
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f6a79d05
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: f6a79d05a81c6096c13417ce48ba1a8e29ed4f90
Parents: 87c51ae 72f1bea
Author: Todd Nine <to...@apache.org>
Authored: Tue Sep 30 14:03:42 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Tue Sep 30 14:03:42 2014 -0600
----------------------------------------------------------------------
.../apache/usergrid/services/notifications/QueueListener.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[11/35] git commit: Hack to ensure that order by works for String and
Number fields.
Posted by sn...@apache.org.
Hack to ensure that order by works for String and Number fields.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/68ee7263
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/68ee7263
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/68ee7263
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 68ee726389200456fb462275c2c603c2c1b5d395
Parents: 16cc4df
Author: Dave Johnson <dm...@apigee.com>
Authored: Sun Sep 28 11:44:55 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sun Sep 28 11:44:55 2014 -0400
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/68ee7263/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 5fb787c..4843854 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -57,7 +57,10 @@ import org.apache.usergrid.persistence.model.field.SetField;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
+import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
+import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
@@ -402,18 +405,29 @@ public class EsEntityIndexImpl implements EntityIndex {
srb = srb.setFrom(0).setSize(query.getLimit());
for (Query.SortPredicate sp : query.getSortPredicates()) {
+
final SortOrder order;
if (sp.getDirection().equals(Query.SortDirection.ASCENDING)) {
order = SortOrder.ASC;
} else {
order = SortOrder.DESC;
}
+
+ String stringFieldName = STRING_PREFIX + sp.getPropertyName();
FieldSortBuilder sort = SortBuilders
- .fieldSort(sp.getPropertyName())
+ .fieldSort( stringFieldName )
+ .order(order)
+ .ignoreUnmapped(true);
+ srb.addSort( sort );
+ log.debug(" Sort: {} order by {}", stringFieldName, order.toString());
+
+ String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
+ sort = SortBuilders
+ .fieldSort( numberFieldName )
.order(order)
.ignoreUnmapped(true);
srb.addSort( sort );
- log.debug(" Sort: {} order by {}", sp.getPropertyName(), order.toString());
+ log.debug(" Sort: {} order by {}", numberFieldName, order.toString());
}
searchResponse = srb.execute().actionGet();
@@ -437,8 +451,6 @@ public class EsEntityIndexImpl implements EntityIndex {
SearchHits hits = searchResponse.getHits();
log.debug(" Hit count: {} Total hits: {}", hits.getHits().length, hits.getTotalHits() );
- // TODO: do we always want to fetch entities? When do we fetch refs or ids?
- // list of entities that will be returned
List<CandidateResult> candidates = new ArrayList<CandidateResult>();
for (SearchHit hit : hits.getHits()) {
[22/35] git commit: Fixes spring configuration of queue listener
Posted by sn...@apache.org.
Fixes spring configuration of queue listener
Fixes bug where executor pool size is always 1. Now matches number of threads so that more than 1 task will actually be running.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9e2743d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9e2743d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9e2743d0
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 9e2743d0debe99f044f2b36105feb364fbe9325c
Parents: cbea83c
Author: Todd Nine <to...@apache.org>
Authored: Mon Sep 29 12:02:24 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Mon Sep 29 12:02:24 2014 -0600
----------------------------------------------------------------------
.../services/notifications/QueueListener.java | 29 ++++++++++----------
.../resources/usergrid-services-context.xml | 9 ++++--
.../apns/NotificationsServiceIT.java | 2 +-
.../gcm/NotificationsServiceIT.java | 4 +--
4 files changed, 24 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e2743d0/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index d8acdfe..42f9dc4 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -40,16 +40,14 @@ public class QueueListener {
private static final Logger LOG = LoggerFactory.getLogger(QueueListener.class);
- @Autowired
+
private MetricsFactory metricsService;
- @Autowired
private ServiceManagerFactory smf;
- @Autowired
private EntityManagerFactory emf;
- @Autowired
+
private Properties properties;
private org.apache.usergrid.mq.QueueManager queueManager;
@@ -60,18 +58,16 @@ public class QueueListener {
private long sleepBetweenRuns = 5000;
- ExecutorService pool;
- List<Future> futures;
+ private ExecutorService pool;
+ private List<Future> futures;
public final String MAX_THREADS = "2";
private Integer batchSize = 1000;
private String[] queueNames;
- public QueueListener() {
- pool = Executors.newFixedThreadPool(1);
- }
+
+
public QueueListener(ServiceManagerFactory smf, EntityManagerFactory emf, MetricsFactory metricsService, Properties props){
- this();
this.smf = smf;
this.emf = emf;
this.metricsService = metricsService;
@@ -79,11 +75,7 @@ public class QueueListener {
}
@PostConstruct
- void init() {
- run();
- }
-
- public void run(){
+ public void start(){
boolean shouldRun = new Boolean(properties.getProperty("usergrid.notifications.listener.run", "true"));
if(shouldRun) {
@@ -98,6 +90,11 @@ public class QueueListener {
int maxThreads = new Integer(properties.getProperty("usergrid.notifications.listener.maxThreads", MAX_THREADS));
futures = new ArrayList<Future>(maxThreads);
+
+ //create our thread pool based on our threadcount.
+
+ pool = Executors.newFixedThreadPool(maxThreads);
+
while (threadCount++ < maxThreads) {
LOG.info("QueueListener: Starting thread {}.", threadCount);
Runnable task = new Runnable() {
@@ -212,6 +209,8 @@ public class QueueListener {
for(Future future : futures){
future.cancel(true);
}
+
+ pool.shutdownNow();
}
private QueueResults getDeliveryBatch(org.apache.usergrid.mq.QueueManager queueManager,String queuePath) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e2743d0/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index be47f08..56883fa 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -89,7 +89,12 @@
<bean id="exportJob" class="org.apache.usergrid.management.export.ExportJob" />
- <bean id="notificationsQueueListener" class="org.apache.usergrid.services.notifications.QueueListener" scope="singleton"/>
-
+ <bean id="notificationsQueueListener" class="org.apache.usergrid.services.notifications.QueueListener"
+ scope="singleton">
+ <constructor-arg name="emf" ref="entityManagerFactory" />
+ <constructor-arg name="metricsService" ref="metricsFactory" />
+ <constructor-arg name="props" ref="properties" />
+ <constructor-arg name="smf" ref="serviceManagerFactory" />
+ </bean>
</beans>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e2743d0/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
index 1985b19..8d994e9 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
@@ -132,7 +132,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
listener = new QueueListener(ns.getServiceManagerFactory(),ns.getEntityManagerFactory(),ns.getMetricsFactory(), new Properties());
listener.DEFAULT_SLEEP = 200;
- listener.run();
+ listener.start();
}
@After
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9e2743d0/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index fdcf7b6..c540c5a 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -108,7 +108,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
ns.getEntityManagerFactory(),ns.getMetricsFactory(), new Properties());
listener.DEFAULT_SLEEP = 200;
- listener.run();
+ listener.start();
}
@After
@@ -571,4 +571,4 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
// assertNull(device2.getProperty(notifier.getName() +
// NOTIFIER_ID_POSTFIX));
// }
-}
\ No newline at end of file
+}
[34/35] git commit: Fixes issue with range scanning and swallows
exception.
Posted by sn...@apache.org.
Fixes issue with range scanning and swallows exception.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/abbd76eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/abbd76eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/abbd76eb
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: abbd76eb40328f63e79441d404555e85894d86fe
Parents: e5bcbb2
Author: Todd Nine <to...@apache.org>
Authored: Tue Sep 30 17:56:02 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Tue Sep 30 17:56:02 2014 -0600
----------------------------------------------------------------------
.../mq/cassandra/io/AbstractSearch.java | 60 +++++++++++++++-----
.../mq/cassandra/io/ConsumerTransaction.java | 2 +-
2 files changed, 48 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbd76eb/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
index ffda843..0e7dea1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
@@ -30,6 +30,8 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.thrift.InvalidRequestException;
+
import org.apache.usergrid.mq.Message;
import org.apache.usergrid.mq.QueueResults;
import org.apache.usergrid.mq.cassandra.io.NoTransactionSearch.SearchParam;
@@ -37,13 +39,12 @@ import org.apache.usergrid.persistence.exceptions.QueueException;
import org.apache.usergrid.persistence.hector.CountingMutator;
import org.apache.usergrid.utils.UUIDUtils;
-import com.fasterxml.uuid.UUIDComparator;
-
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
import me.prettyprint.hector.api.beans.HColumn;
import me.prettyprint.hector.api.beans.Row;
import me.prettyprint.hector.api.beans.Rows;
+import me.prettyprint.hector.api.exceptions.HInvalidRequestException;
import me.prettyprint.hector.api.factory.HFactory;
import me.prettyprint.hector.api.mutation.Mutator;
import me.prettyprint.hector.api.query.SliceQuery;
@@ -65,7 +66,6 @@ import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.persistence.cassandra.Serializers.se;
import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import static org.apache.usergrid.utils.NumberUtils.roundLong;
-import static org.apache.usergrid.utils.UUIDUtils.compare;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMillis;
@@ -156,7 +156,7 @@ public abstract class AbstractSearch implements QueueSearch {
UUID finish_uuid = params.reversed ? bounds.getOldest() : bounds.getNewest();
- List<UUID> results = new ArrayList<UUID>( params.limit );
+ List<UUID> results = new ArrayList<>( params.limit );
UUID start = params.startId;
@@ -184,7 +184,7 @@ public abstract class AbstractSearch implements QueueSearch {
current_ts_shard = finish_ts_shard;
}
- final MessageIdComparator comparator = new MessageIdComparator(params.reversed);
+ final MessageIdComparator comparator = new MessageIdComparator( params.reversed );
//should be start < finish
@@ -196,14 +196,14 @@ public abstract class AbstractSearch implements QueueSearch {
}
-
-
UUID lastValue = start;
boolean firstPage = true;
- while ( ( current_ts_shard >= start_ts_shard ) && ( current_ts_shard <= finish_ts_shard ) && comparator.compare(start, finish_uuid) < 1 ) {
+ while ( ( current_ts_shard >= start_ts_shard ) && ( current_ts_shard <= finish_ts_shard )
+ && comparator.compare( start, finish_uuid ) < 1 ) {
- logger.info( "Starting search with start UUID {}, finish UUID {}, and reversed {}", new Object[]{lastValue, finish_uuid, params.reversed });
+ logger.info( "Starting search with start UUID {}, finish UUID {}, and reversed {}",
+ new Object[] { lastValue, finish_uuid, params.reversed } );
SliceQuery<ByteBuffer, UUID, ByteBuffer> q = createSliceQuery( ko, be, ue, be );
@@ -211,7 +211,8 @@ public abstract class AbstractSearch implements QueueSearch {
q.setKey( getQueueShardRowKey( queueId, current_ts_shard ) );
q.setRange( lastValue, finish_uuid, params.reversed, params.limit + 1 );
- final List<HColumn<UUID, ByteBuffer>> cassResults = q.execute().get().getColumns();
+ final List<HColumn<UUID, ByteBuffer>> cassResults = swallowOrderedExecution(q);
+
for ( int i = 0; i < cassResults.size(); i++ ) {
HColumn<UUID, ByteBuffer> column = cassResults.get( i );
@@ -339,12 +340,12 @@ public abstract class AbstractSearch implements QueueSearch {
}
- private static final class MessageIdComparator implements Comparator<UUID> {
+ protected static final class MessageIdComparator implements Comparator<UUID> {
private final int comparator;
- private MessageIdComparator( final boolean reversed ) {
+ protected MessageIdComparator( final boolean reversed ) {
this.comparator = reversed ? -1 : 1;
}
@@ -352,7 +353,40 @@ public abstract class AbstractSearch implements QueueSearch {
@Override
public int compare( final UUID o1, final UUID o2 ) {
- return UUIDUtils.compare( o1, o2 )*comparator;
+ return UUIDUtils.compare( o1, o2 ) * comparator;
+ }
+ }
+
+
+ /**
+ * This method intentionally swallows ordered execution issues. For some reason, our Time UUID ordering does
+ * not agree with the cassandra comparator as our micros get very close
+ * @param query
+ * @param <K>
+ * @param <UUID>
+ * @param <V>
+ * @return
+ */
+ protected static <K, UUID, V> List<HColumn<UUID, V>> swallowOrderedExecution( final SliceQuery<K, UUID, V> query ) {
+ try {
+
+ return query.execute().get().getColumns();
+ }
+ catch ( HInvalidRequestException e ) {
+ //invalid request. Occasionally we get order issues when there shouldn't be, disregard them.
+
+ final Throwable invalidRequestException = e.getCause();
+
+ if ( invalidRequestException instanceof InvalidRequestException
+ //we had a range error
+ && ( ( InvalidRequestException ) invalidRequestException ).getWhy().contains(
+ "range finish must come after start in the order of traversal" )) {
+ return Collections.emptyList();
+ }
+
+ throw e;
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/abbd76eb/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
index 7205dbc..bdf9afd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
@@ -337,7 +337,7 @@ public class ConsumerTransaction extends NoTransactionSearch
q.setKey( getQueueClientTransactionKey( queueId, consumerId ) );
q.setRange( params.startId, startTimeUUID, false, params.limit + 1 );
- List<HColumn<UUID, UUID>> cassResults = q.execute().get().getColumns();
+ List<HColumn<UUID, UUID>> cassResults = swallowOrderedExecution(q);
List<TransactionPointer> results = new ArrayList<TransactionPointer>( params.limit );
[28/35] git commit: Adds additional logging to help diagnose queue
issue
Posted by sn...@apache.org.
Adds additional logging to help diagnose queue issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9ce83461
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9ce83461
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9ce83461
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 9ce83461b7d6f4eaaec10b14ed2c50f57b7629bf
Parents: d6fd7dd
Author: Todd Nine <to...@apache.org>
Authored: Tue Sep 30 12:26:01 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Tue Sep 30 12:26:01 2014 -0600
----------------------------------------------------------------------
.../usergrid/mq/cassandra/QueueManagerImpl.java | 18 ++-
.../mq/cassandra/io/AbstractSearch.java | 141 +++++++++----------
.../mq/cassandra/io/ConsumerTransaction.java | 2 +-
3 files changed, 80 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ce83461/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueueManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueueManagerImpl.java
index 57a6e3b..ae4e4c9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueueManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/QueueManagerImpl.java
@@ -192,18 +192,24 @@ public class QueueManagerImpl implements QueueManager {
long shard_ts = roundLong( message.getTimestamp(), QUEUE_SHARD_INTERVAL );
- logger.debug( "Adding message with id '{}' to queue '{}'", message.getUuid(), queueId );
+ final UUID messageUuid = message.getUuid();
+
+
+ logger.debug( "Adding message with id '{}' to queue '{}'", messageUuid, queueId );
+
batch.addInsertion( getQueueShardRowKey( queueId, shard_ts ), QUEUE_INBOX.getColumnFamily(),
- createColumn( message.getUuid(), ByteBuffer.allocate( 0 ), timestamp, ue, be ) );
+ createColumn( messageUuid, ByteBuffer.allocate( 0 ), timestamp, ue, be ) );
- long oldest_ts = Long.MAX_VALUE - getTimestampInMicros( message.getUuid() );
+ long oldest_ts = Long.MAX_VALUE - getTimestampInMicros( messageUuid );
batch.addInsertion( bytebuffer( queueId ), QUEUE_PROPERTIES.getColumnFamily(),
- createColumn( QUEUE_OLDEST, message.getUuid(), oldest_ts, se, ue ) );
+ createColumn( QUEUE_OLDEST, messageUuid, oldest_ts, se, ue ) );
- long newest_ts = getTimestampInMicros( message.getUuid() );
+ long newest_ts = getTimestampInMicros( messageUuid );
batch.addInsertion( bytebuffer( queueId ), QUEUE_PROPERTIES.getColumnFamily(),
- createColumn( QUEUE_NEWEST, message.getUuid(), newest_ts, se, ue ) );
+ createColumn( QUEUE_NEWEST, messageUuid, newest_ts, se, ue ) );
+
+ logger.debug( "Writing UUID {} with oldest timestamp {} and newest with timestamp {}", new Object[]{messageUuid, oldest_ts, newest_ts});
batch.addInsertion( bytebuffer( getQueueId( "/" ) ), QUEUE_SUBSCRIBERS.getColumnFamily(),
createColumn( queuePath, queueId, timestamp, se, ue ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ce83461/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
index c3d2800..b0b5ac7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
@@ -29,6 +29,7 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.mq.Message;
import org.apache.usergrid.mq.QueueResults;
import org.apache.usergrid.mq.cassandra.io.NoTransactionSearch.SearchParam;
@@ -47,7 +48,6 @@ import me.prettyprint.hector.api.query.SliceQuery;
import static me.prettyprint.hector.api.factory.HFactory.createColumn;
import static me.prettyprint.hector.api.factory.HFactory.createMultigetSliceQuery;
-
import static me.prettyprint.hector.api.factory.HFactory.createSliceQuery;
import static org.apache.usergrid.mq.Queue.QUEUE_NEWEST;
import static org.apache.usergrid.mq.Queue.QUEUE_OLDEST;
@@ -59,16 +59,15 @@ import static org.apache.usergrid.mq.cassandra.QueuesCF.CONSUMERS;
import static org.apache.usergrid.mq.cassandra.QueuesCF.MESSAGE_PROPERTIES;
import static org.apache.usergrid.mq.cassandra.QueuesCF.QUEUE_INBOX;
import static org.apache.usergrid.mq.cassandra.QueuesCF.QUEUE_PROPERTIES;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
+import static org.apache.usergrid.persistence.cassandra.Serializers.se;
+import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import static org.apache.usergrid.utils.NumberUtils.roundLong;
-import static org.apache.usergrid.utils.UUIDUtils.MAX_TIME_UUID;
-import static org.apache.usergrid.utils.UUIDUtils.MIN_TIME_UUID;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMillis;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
/** @author tnine */
-public abstract class AbstractSearch implements QueueSearch
-{
+public abstract class AbstractSearch implements QueueSearch {
private static final Logger logger = LoggerFactory.getLogger( AbstractSearch.class );
@@ -78,8 +77,7 @@ public abstract class AbstractSearch implements QueueSearch
/**
*
*/
- public AbstractSearch( Keyspace ko )
- {
+ public AbstractSearch( Keyspace ko ) {
this.ko = ko;
}
@@ -90,13 +88,11 @@ public abstract class AbstractSearch implements QueueSearch
* @param queueId The queueId
* @param consumerId The consumerId
*/
- public UUID getConsumerQueuePosition( UUID queueId, UUID consumerId )
- {
+ public UUID getConsumerQueuePosition( UUID queueId, UUID consumerId ) {
HColumn<UUID, UUID> result =
HFactory.createColumnQuery( ko, ue, ue, ue ).setKey( consumerId ).setName( queueId )
.setColumnFamily( CONSUMERS.getColumnFamily() ).execute().get();
- if ( result != null )
- {
+ if ( result != null ) {
return result.getValue();
}
@@ -105,21 +101,19 @@ public abstract class AbstractSearch implements QueueSearch
/** Load the messages into an array list */
- protected List<Message> loadMessages( Collection<UUID> messageIds, boolean reversed )
- {
+ protected List<Message> loadMessages( Collection<UUID> messageIds, boolean reversed ) {
Rows<UUID, String, ByteBuffer> messageResults =
createMultigetSliceQuery( ko, ue, se, be ).setColumnFamily( MESSAGE_PROPERTIES.getColumnFamily() )
- .setKeys( messageIds ).setRange( null, null, false, ALL_COUNT ).execute().get();
+ .setKeys( messageIds )
+ .setRange( null, null, false, ALL_COUNT ).execute().get();
List<Message> messages = new ArrayList<Message>( messageIds.size() );
- for ( Row<UUID, String, ByteBuffer> row : messageResults )
- {
+ for ( Row<UUID, String, ByteBuffer> row : messageResults ) {
Message message = deserializeMessage( row.getColumnSlice().getColumns() );
- if ( message != null )
- {
+ if ( message != null ) {
messages.add( message );
}
}
@@ -131,13 +125,11 @@ public abstract class AbstractSearch implements QueueSearch
/** Create the results to return from the given messages */
- protected QueueResults createResults( List<Message> messages, String queuePath, UUID queueId, UUID consumerId )
- {
+ protected QueueResults createResults( List<Message> messages, String queuePath, UUID queueId, UUID consumerId ) {
UUID lastId = null;
- if ( messages != null && messages.size() > 0 )
- {
+ if ( messages != null && messages.size() > 0 ) {
lastId = messages.get( messages.size() - 1 ).getUuid();
}
@@ -152,11 +144,9 @@ public abstract class AbstractSearch implements QueueSearch
* @param queueId The queue id to read
* @param bounds The bounds to use when reading
*/
- protected List<UUID> getQueueRange( UUID queueId, QueueBounds bounds, SearchParam params )
- {
+ protected List<UUID> getQueueRange( UUID queueId, QueueBounds bounds, SearchParam params ) {
- if ( bounds == null )
- {
+ if ( bounds == null ) {
logger.error( "Necessary queue bounds not found" );
throw new QueueException( "Neccessary queue bounds not found" );
}
@@ -167,19 +157,16 @@ public abstract class AbstractSearch implements QueueSearch
UUID start = params.startId;
- if ( start == null )
- {
+ if ( start == null ) {
start = params.reversed ? bounds.getNewest() : bounds.getOldest();
}
- if ( start == null )
- {
+ if ( start == null ) {
logger.error( "No first message in queue" );
return results;
}
- if ( finish_uuid == null )
- {
+ if ( finish_uuid == null ) {
logger.error( "No last message in queue" );
return results;
}
@@ -189,51 +176,68 @@ public abstract class AbstractSearch implements QueueSearch
long finish_ts_shard = roundLong( getTimestampInMillis( finish_uuid ), QUEUE_SHARD_INTERVAL );
long current_ts_shard = start_ts_shard;
- if ( params.reversed )
- {
+
+ if ( params.reversed ) {
current_ts_shard = finish_ts_shard;
}
+ //should be start < finish
+ if ( !params.reversed && UUIDUtils.compare( start, finish_uuid ) > 0 ) {
+ logger.warn( "Tried to perform a slice with start UUID {} after finish UUID {}.", start, finish_uuid );
+ throw new IllegalArgumentException( String.format("You cannot specify a start value of %s after finish value of %s", start, finish_uuid) );
+ }
+
+ // should be finish < start
+ else if ( params.reversed && UUIDUtils.compare( start, finish_uuid ) < 0 ) {
+ logger.warn( "Tried to perform a slice with start UUID {} after finish UUID {}.", start, finish_uuid );
+ throw new IllegalArgumentException( String.format("You cannot specify a start value of %s after finish value of %s", start, finish_uuid) );
+ }
+
+
+
+ UUID lastValue = start;
+ boolean firstPage = true;
+
+ while ( ( current_ts_shard >= start_ts_shard ) && ( current_ts_shard <= finish_ts_shard ) ) {
- while ( ( current_ts_shard >= start_ts_shard ) && ( current_ts_shard <= finish_ts_shard ) )
- {
SliceQuery<ByteBuffer, UUID, ByteBuffer> q = createSliceQuery( ko, be, ue, be );
q.setColumnFamily( QUEUE_INBOX.getColumnFamily() );
q.setKey( getQueueShardRowKey( queueId, current_ts_shard ) );
- q.setRange( start, finish_uuid, params.reversed, params.limit + 1 );
+ q.setRange( lastValue, finish_uuid, params.reversed, params.limit + 1 );
- List<HColumn<UUID, ByteBuffer>> cassResults = q.execute().get().getColumns();
+ final List<HColumn<UUID, ByteBuffer>> cassResults = q.execute().get().getColumns();
- for ( int i = 0; i < cassResults.size(); i++ )
- {
+ for ( int i = 0; i < cassResults.size(); i++ ) {
HColumn<UUID, ByteBuffer> column = cassResults.get( i );
+ final UUID columnName = column.getName();
+
// skip the first one, we've already read it
- if ( i == 0 && params.skipFirst && params.startId.equals( column.getName() ) )
- {
+ if ( i == 0 && (firstPage && params.skipFirst && params.startId.equals( columnName ))
+ || (!firstPage && lastValue != null && lastValue.equals(columnName)) ) {
continue;
}
- UUID id = column.getName();
- results.add( id );
+ lastValue = columnName;
+
+ results.add( columnName );
- logger.debug( "Added id '{}' to result set for queue id '{}'", id, queueId );
+ logger.debug( "Added id '{}' to result set for queue id '{}'", start, queueId );
- if ( results.size() >= params.limit )
- {
+ if ( results.size() >= params.limit ) {
return results;
}
+
+ firstPage = false;
}
- if ( params.reversed )
- {
+ if ( params.reversed ) {
current_ts_shard -= QUEUE_SHARD_INTERVAL;
}
- else
- {
+ else {
current_ts_shard += QUEUE_SHARD_INTERVAL;
}
}
@@ -247,23 +251,19 @@ public abstract class AbstractSearch implements QueueSearch
*
* @return The bounds for the queue
*/
- public QueueBounds getQueueBounds( UUID queueId )
- {
- try
- {
+ public QueueBounds getQueueBounds( UUID queueId ) {
+ try {
ColumnSlice<String, UUID> result = HFactory.createSliceQuery( ko, ue, se, ue ).setKey( queueId )
.setColumnNames( QUEUE_NEWEST, QUEUE_OLDEST )
.setColumnFamily( QUEUE_PROPERTIES.getColumnFamily() ).execute()
.get();
if ( result != null && result.getColumnByName( QUEUE_OLDEST ) != null
- && result.getColumnByName( QUEUE_NEWEST ) != null )
- {
+ && result.getColumnByName( QUEUE_NEWEST ) != null ) {
return new QueueBounds( result.getColumnByName( QUEUE_OLDEST ).getValue(),
result.getColumnByName( QUEUE_NEWEST ).getValue() );
}
}
- catch ( Exception e )
- {
+ catch ( Exception e ) {
logger.error( "Error getting oldest queue message ID", e );
}
return null;
@@ -276,11 +276,9 @@ public abstract class AbstractSearch implements QueueSearch
* @param lastReturnedId This is a null safe parameter. If it's null, this won't be written since it means we didn't
* read any messages
*/
- protected void writeClientPointer( UUID queueId, UUID consumerId, UUID lastReturnedId )
- {
+ protected void writeClientPointer( UUID queueId, UUID consumerId, UUID lastReturnedId ) {
// nothing to do
- if ( lastReturnedId == null )
- {
+ if ( lastReturnedId == null ) {
return;
}
@@ -292,8 +290,7 @@ public abstract class AbstractSearch implements QueueSearch
Mutator<UUID> mutator = CountingMutator.createFlushingMutator( ko, ue );
- if ( logger.isDebugEnabled() )
- {
+ if ( logger.isDebugEnabled() ) {
logger.debug( "Writing last client id pointer of '{}' for queue '{}' and consumer '{}' with timestamp '{}",
new Object[] {
lastReturnedId, queueId, consumerId, colTimestamp
@@ -307,18 +304,15 @@ public abstract class AbstractSearch implements QueueSearch
}
- private class RequestedOrderComparator implements Comparator<Message>
- {
+ private class RequestedOrderComparator implements Comparator<Message> {
private Map<UUID, Integer> indexCache = new HashMap<UUID, Integer>();
- private RequestedOrderComparator( Collection<UUID> ids )
- {
+ private RequestedOrderComparator( Collection<UUID> ids ) {
int i = 0;
- for ( UUID id : ids )
- {
+ for ( UUID id : ids ) {
indexCache.put( id, i );
i++;
}
@@ -331,8 +325,7 @@ public abstract class AbstractSearch implements QueueSearch
* @see java.util.Comparator#compare(java.lang.Object, java.lang.Object)
*/
@Override
- public int compare( Message o1, Message o2 )
- {
+ public int compare( Message o1, Message o2 ) {
int o1Idx = indexCache.get( o1.getUuid() );
int o2Idx = indexCache.get( o2.getUuid() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9ce83461/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
index 7e45b00..6d70924 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
@@ -230,7 +230,7 @@ public class ConsumerTransaction extends NoTransactionSearch
List<TransactionPointer> pointers = getConsumerIds( queueId, consumerId, params, startTimeUUID );
- TransactionPointer pointer = null;
+ TransactionPointer pointer;
int lastTransactionIndex = -1;
[20/35] git commit: Ignoring tests that are causing build to fail.
Needs to be run with external cassandra or manually in Jenkins
Posted by sn...@apache.org.
Ignoring tests that are causing build to fail. Needs to be run with external cassandra or manually in Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/deef46be
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/deef46be
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/deef46be
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: deef46be1018373c825b448486eaa5418c0b9a2b
Parents: 3748113
Author: Todd Nine <to...@apache.org>
Authored: Mon Sep 29 12:01:10 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Mon Sep 29 12:01:10 2014 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/PerformanceEntityReadTest.java | 2 ++
.../apache/usergrid/persistence/PerformanceEntityWriteTest.java | 2 ++
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/deef46be/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityReadTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityReadTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityReadTest.java
index 41a5dea..03d9fa8 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityReadTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityReadTest.java
@@ -24,6 +24,7 @@ import java.util.concurrent.TimeUnit;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
@@ -44,6 +45,7 @@ import org.apache.usergrid.cassandra.Concurrent;
//@RunWith(JukitoRunner.class)
//@UseModules({ GuiceModule.class })
@Concurrent()
+@Ignore("Kills embedded cassandra")
public class PerformanceEntityReadTest extends AbstractCoreIT {
private static final Logger logger = LoggerFactory.getLogger(PerformanceEntityReadTest.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/deef46be/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java
index 1b7c96d..9dfa3c4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityWriteTest.java
@@ -25,6 +25,7 @@ import java.util.concurrent.TimeUnit;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
@@ -43,6 +44,7 @@ import com.codahale.metrics.Slf4jReporter;
//@RunWith(JukitoRunner.class)
//@UseModules({ GuiceModule.class })
@Concurrent()
+@Ignore("Kills embedded cassandra")
public class PerformanceEntityWriteTest extends AbstractCoreIT {
private static final Logger LOG = LoggerFactory.getLogger( PerformanceEntityWriteTest.class );
[09/35] git commit: more logging, max time to wait on error
Posted by sn...@apache.org.
more logging, max time to wait on error
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/86edd643
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/86edd643
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/86edd643
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 86edd64342b4049825fa8f8597887a45d24ceb8b
Parents: 38eb9e4
Author: Shawn Feldman <sf...@apache.org>
Authored: Sat Sep 27 11:35:42 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Sat Sep 27 11:35:42 2014 -0600
----------------------------------------------------------------------
.../usergrid/services/notifications/ApplicationQueueManager.java | 2 +-
.../apache/usergrid/services/notifications/QueueListener.java | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86edd643/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
index dac3b6a..0f76dc1 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
@@ -180,7 +180,7 @@ public class ApplicationQueueManager implements QueueManager {
}
now = System.currentTimeMillis();
qm.postToQueue(queueName, message);
- LOG.info("ApplicationQueueMessage: notification {} post-queue to device {} duration " + (System.currentTimeMillis() - now) + " ms", notification.getUuid(), deviceRef.getUuid());
+ LOG.info("ApplicationQueueMessage: notification {} post-queue to device {} duration " + (System.currentTimeMillis() - now) + " ms "+queueName+" queue", notification.getUuid(), deviceRef.getUuid());
deviceCount.incrementAndGet();
queueMeter.mark();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/86edd643/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index 6281ef9..056c3ca 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -193,7 +193,9 @@ public class QueueListener {
}catch (Exception ex){
LOG.error("failed to dequeue",ex);
try {
- long sleeptime = sleepWhenNoneFound*(consecutiveExceptions.incrementAndGet());
+ long sleeptime = sleepWhenNoneFound*consecutiveExceptions.incrementAndGet();
+ long maxSleep = 15000;
+ sleeptime = sleeptime > maxSleep ? maxSleep : sleeptime ;
LOG.info("sleeping due to failures {} ms", sleeptime);
Thread.sleep(sleeptime);
}catch (InterruptedException ie){
[24/35] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a05ed4ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a05ed4ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a05ed4ec
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: a05ed4ec729795ec7de68ac056aea1f368a55b78
Parents: 4f68f72 9e2743d
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Sep 29 14:10:22 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Sep 29 14:10:22 2014 -0400
----------------------------------------------------------------------
.../mq/cassandra/io/AbstractSearch.java | 17 ++--------
.../persistence/PerformanceEntityReadTest.java | 2 ++
.../persistence/PerformanceEntityWriteTest.java | 2 ++
.../services/notifications/QueueListener.java | 34 ++++++++++----------
.../resources/usergrid-services-context.xml | 9 ++++--
.../apns/NotificationsServiceIT.java | 2 +-
.../gcm/NotificationsServiceIT.java | 4 +--
7 files changed, 34 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
[35/35] git commit: Merge branch 'two-dot-o' into
two-dot-o-rebuildable-index
Posted by sn...@apache.org.
Merge branch 'two-dot-o' into two-dot-o-rebuildable-index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3df5d4d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3df5d4d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3df5d4d2
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 3df5d4d25b83e39dfd1fbe0919bc68d3d84ed821
Parents: 1aa04a7 abbd76e
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Oct 1 10:51:03 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Oct 1 10:51:03 2014 -0400
----------------------------------------------------------------------
portal/js/global/ug-service.js | 10 +-
.../corepersistence/CpEntityManager.java | 3 +-
.../corepersistence/CpRelationManager.java | 6 +-
.../usergrid/mq/cassandra/QueueManagerImpl.java | 18 +-
.../mq/cassandra/io/AbstractSearch.java | 208 +++++++++++--------
.../mq/cassandra/io/ConsumerTransaction.java | 19 +-
.../persistence/PerformanceEntityReadTest.java | 2 +
.../persistence/PerformanceEntityWriteTest.java | 2 +
.../index/impl/EsEntityIndexImpl.java | 52 ++++-
.../persistence/index/impl/EsQueryVistor.java | 45 ++--
.../java/org/apache/usergrid/rest/BasicIT.java | 34 ++-
.../notifications/ApplicationQueueManager.java | 40 ++--
.../notifications/NotificationsService.java | 7 +-
.../notifications/NotificationsTaskManager.java | 33 ---
.../services/notifications/QueueListener.java | 64 +++---
.../services/notifications/QueueManager.java | 4 -
.../notifications/SingleQueueTaskManager.java | 10 +-
.../services/notifications/TaskTracker.java | 4 +-
.../resources/usergrid-services-context.xml | 9 +-
.../apns/NotificationsServiceIT.java | 6 +-
.../gcm/NotificationsServiceIT.java | 9 +-
21 files changed, 342 insertions(+), 243 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3df5d4d2/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3df5d4d2/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
[27/35] git commit: update queue numbers
Posted by sn...@apache.org.
update queue numbers
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d6fd7dd3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d6fd7dd3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d6fd7dd3
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: d6fd7dd3009728a25b84ad87dcf9819587fdd758
Parents: 48a3af1
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Sep 29 16:24:34 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Sep 29 16:24:34 2014 -0600
----------------------------------------------------------------------
.../usergrid/services/notifications/ApplicationQueueManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d6fd7dd3/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
index a9491d0..cde14a5 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
@@ -47,7 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public class ApplicationQueueManager implements QueueManager {
- public static String DEFAULT_QUEUE_NAME = "notifications/queuelistenerv1_23;notifications/queuelistenerv1_24;notifications/queuelistenerv1_25";
+ public static String DEFAULT_QUEUE_NAME = "notifications/queuelistenerv1_34;notifications/queuelistenerv1_35;notifications/queuelistenerv1_36";
public static final String DEFAULT_QUEUE_PROPERTY = "usergrid.notifications.listener.queueName";
private static final Logger LOG = LoggerFactory.getLogger(ApplicationQueueManager.class);
[06/35] git commit: Fixes USERGRID-233 which turns out to be a merge
problem.
Posted by sn...@apache.org.
Fixes USERGRID-233 which turns out to be a merge problem.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/43b85a61
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/43b85a61
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/43b85a61
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 43b85a618ddef4ca605cb06153f92a55aa34d567
Parents: 4cba3b8
Author: Dave Johnson <dm...@apigee.com>
Authored: Sat Sep 27 10:55:42 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sat Sep 27 10:55:42 2014 -0400
----------------------------------------------------------------------
portal/js/global/ug-service.js | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/43b85a61/portal/js/global/ug-service.js
----------------------------------------------------------------------
diff --git a/portal/js/global/ug-service.js b/portal/js/global/ug-service.js
index f3cba06..db2bfe9 100644
--- a/portal/js/global/ug-service.js
+++ b/portal/js/global/ug-service.js
@@ -550,13 +550,8 @@ AppServices.Services.factory('ug', function(configuration, $rootScope, utility,
self.notificationCollection = notifications;
});
},
-<<<<<<< HEAD
- getNotificationReceipts: function (uuid) {
- this.getCollection('receipts', 'receipts','created desc','notificationUUID='+uuid);
-=======
getNotificationReceipts: function(uuid) {
this.getCollection('receipts', 'notifications/' + uuid + '/receipts');
->>>>>>> d972699... fix create collections
var self = this;
$rootScope.$on('receipts-received', function(event, receipts) {
self.receiptsCollection = receipts;
[17/35] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fb050ff4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fb050ff4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fb050ff4
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: fb050ff461c8f0b4386252e479c4c72405c6723f
Parents: 49108ef dfebcc1
Author: Dave Johnson <dm...@apigee.com>
Authored: Sun Sep 28 12:37:39 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sun Sep 28 12:37:39 2014 -0400
----------------------------------------------------------------------
.../services/notifications/ApplicationQueueManager.java | 2 +-
.../apache/usergrid/services/notifications/QueueListener.java | 6 +++---
.../services/notifications/apns/NotificationsServiceIT.java | 2 +-
.../services/notifications/gcm/NotificationsServiceIT.java | 3 ++-
4 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[04/35] git commit: move notification update
Posted by sn...@apache.org.
move notification update
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/96ea0293
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/96ea0293
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/96ea0293
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 96ea029350edf68fa0972127d943a3f3d7775b8c
Parents: 05f56db
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Sep 26 16:44:04 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Sep 26 16:44:04 2014 -0600
----------------------------------------------------------------------
.../usergrid/services/notifications/ApplicationQueueManager.java | 2 ++
.../org/apache/usergrid/services/notifications/JobScheduler.java | 1 -
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96ea0293/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
index 4bd0cde..d9542ed 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
@@ -92,6 +92,7 @@ public class ApplicationQueueManager implements QueueManager {
public void queueNotification(final Notification notification, final JobExecution jobExecution) throws Exception {
if(scheduleQueueJob(notification)){
+ em.update(notification);
return;
}
final Meter queueMeter = metricsFactory.getMeter(ApplicationQueueManager.class,"queue");
@@ -121,6 +122,7 @@ public class ApplicationQueueManager implements QueueManager {
//if there are more pages (defined by PAGE_SIZE) you probably want this to be async, also if this is already a job then don't reschedule
if (iterator instanceof ResultsIterator && ((ResultsIterator) iterator).hasPages() && jobExecution == null) {
jobScheduler.scheduleQueueJob(notification, true);
+ em.update(notification);
return;
}
final CountMinSketch sketch = new CountMinSketch(0.0001,.99,7364181); //add probablistic counter to find dups
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96ea0293/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java
index a9acd95..6015a0a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java
@@ -74,7 +74,6 @@ public class JobScheduler{
SchedulerService scheduler = getSchedulerService();
scheduler.createJob("queueJob", scheduleAt, jobData);
LOG.info("notification {} scheduled for queuing", notification.getUuid());
- em.update(notification);
}
return scheduled;
}
[15/35] git commit: new queues
Posted by sn...@apache.org.
new queues
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/dfebcc19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/dfebcc19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/dfebcc19
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: dfebcc195e2860a02da008ac846955f477a405a9
Parents: d2f2838
Author: Shawn Feldman <sf...@apache.org>
Authored: Sun Sep 28 10:29:03 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Sun Sep 28 10:29:03 2014 -0600
----------------------------------------------------------------------
.../usergrid/services/notifications/ApplicationQueueManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dfebcc19/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
index 0f76dc1..a9491d0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
@@ -47,7 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public class ApplicationQueueManager implements QueueManager {
- public static String DEFAULT_QUEUE_NAME = "notifications/queuelistenerv1_20;notifications/queuelistenerv1_21;notifications/queuelistenerv1_22";
+ public static String DEFAULT_QUEUE_NAME = "notifications/queuelistenerv1_23;notifications/queuelistenerv1_24;notifications/queuelistenerv1_25";
public static final String DEFAULT_QUEUE_PROPERTY = "usergrid.notifications.listener.queueName";
private static final Logger LOG = LoggerFactory.getLogger(ApplicationQueueManager.class);
[29/35] git commit: more logging
Posted by sn...@apache.org.
more logging
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c8547dd8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c8547dd8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c8547dd8
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: c8547dd8d0348864303e26370efabb0975b4f866
Parents: 9ce8346
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Sep 30 12:31:00 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Sep 30 12:31:00 2014 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/services/notifications/QueueListener.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c8547dd8/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index 31df5e7..66edb78 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -217,8 +217,9 @@ public class QueueListener {
QueueQuery qq = new QueueQuery();
qq.setLimit(this.getBatchSize());
qq.setTimeout(MESSAGE_TRANSACTION_TIMEOUT);
+ LOG.debug("getting from queue {} ", queuePath);
QueueResults results = queueManager.getFromQueue(queuePath, qq);
- LOG.debug("got batch of {} devices", results.size());
+ LOG.debug("got batch of {} devices from queue {} ", results.size(), queuePath);
return results;
}
[25/35] git commit: Need to support order by boolean too.
Posted by sn...@apache.org.
Need to support order by boolean too.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2ea3ee1a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2ea3ee1a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2ea3ee1a
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 2ea3ee1adde4275e32167d84e3076bfdf7b3f7ad
Parents: a05ed4e
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Sep 29 14:50:16 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Sep 29 14:50:16 2014 -0400
----------------------------------------------------------------------
.../persistence/index/impl/EsEntityIndexImpl.java | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2ea3ee1a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8a1233f..8401e13 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -411,6 +411,11 @@ public class EsEntityIndexImpl implements EntityIndex {
order = SortOrder.DESC;
}
+ // we do not know the type of the "order by" property and so we do not know what
+ // type prefix to use. So, here we add an order by clause for every possible type
+ // that you can order by: string, number and boolean and we ask ElasticSearch
+ // to ignore any fields that are not present.
+
final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
final FieldSortBuilder stringSort = SortBuilders
.fieldSort( stringFieldName )
@@ -426,6 +431,14 @@ public class EsEntityIndexImpl implements EntityIndex {
.ignoreUnmapped(true);
srb.addSort( numberSort );
log.debug(" Sort: {} order by {}", numberFieldName, order.toString());
+
+ final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder booleanSort = SortBuilders
+ .fieldSort( booleanFieldName )
+ .order(order)
+ .ignoreUnmapped(true);
+ srb.addSort( booleanSort );
+ log.debug(" Sort: {} order by {}", booleanFieldName, order.toString());
}
searchResponse = srb.execute().actionGet();
[19/35] git commit: renew queue manager on every iteration
Posted by sn...@apache.org.
renew queue manager on every iteration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/37481138
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/37481138
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/37481138
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 3748113845899da468911848fadb9ad3f1598558
Parents: 009a184
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Sep 29 08:30:41 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Sep 29 08:30:41 2014 -0600
----------------------------------------------------------------------
.../apache/usergrid/services/notifications/QueueListener.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/37481138/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index 043bcb2..d8acdfe 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -124,14 +124,15 @@ public class QueueListener {
private void execute(){
Thread.currentThread().setName("Notifications_Processor"+UUID.randomUUID());
- svcMgr = smf.getServiceManager(smf.getManagementAppId());
- queueManager = svcMgr.getQueueManager();
+
final AtomicInteger consecutiveExceptions = new AtomicInteger();
LOG.info("QueueListener: Starting execute process.");
// run until there are no more active jobs
while ( true ) {
try {
+ svcMgr = smf.getServiceManager(smf.getManagementAppId());
+ queueManager = svcMgr.getQueueManager();
String queueName = ApplicationQueueManager.getRandomQueue(queueNames);
QueueResults results = getDeliveryBatch(queueManager,queueName);
LOG.info("QueueListener: retrieved batch of {} messages", results.size());
[14/35] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d2f28381
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d2f28381
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d2f28381
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: d2f28381fd6d4bf2b3946ffd7d96ee9f2c156bef
Parents: c0f377d 3f6d7b4
Author: Shawn Feldman <sf...@apache.org>
Authored: Sun Sep 28 09:49:54 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Sun Sep 28 09:49:54 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 3 ++-
.../index/impl/EsEntityIndexImpl.java | 20 ++++++++++++++++----
2 files changed, 18 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[30/35] git commit: change to info logging
Posted by sn...@apache.org.
change to info logging
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/72f1beae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/72f1beae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/72f1beae
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 72f1beae032ae51255f9c37c2199e429d3b4c7a7
Parents: c8547dd
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Sep 30 13:58:32 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Sep 30 13:58:32 2014 -0600
----------------------------------------------------------------------
.../apache/usergrid/services/notifications/QueueListener.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/72f1beae/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index 66edb78..10ca49f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -131,8 +131,9 @@ public class QueueListener {
svcMgr = smf.getServiceManager(smf.getManagementAppId());
queueManager = svcMgr.getQueueManager();
String queueName = ApplicationQueueManager.getRandomQueue(queueNames);
+ LOG.info("getting from queue {} ", queueName);
QueueResults results = getDeliveryBatch(queueManager,queueName);
- LOG.info("QueueListener: retrieved batch of {} messages", results.size());
+ LOG.info("QueueListener: retrieved batch of {} messages from queue {} ", results.size(),queueName);
List<Message> messages = results.getMessages();
if (messages.size() > 0) {
@@ -217,9 +218,7 @@ public class QueueListener {
QueueQuery qq = new QueueQuery();
qq.setLimit(this.getBatchSize());
qq.setTimeout(MESSAGE_TRANSACTION_TIMEOUT);
- LOG.debug("getting from queue {} ", queuePath);
QueueResults results = queueManager.getFromQueue(queuePath, qq);
- LOG.debug("got batch of {} devices from queue {} ", results.size(), queuePath);
return results;
}
[33/35] git commit: Added guard for extending UUID on failed
transactions beyond the max of the queue.
Posted by sn...@apache.org.
Added guard for extending UUID on failed transactions beyond the max of the queue.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e5bcbb23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e5bcbb23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e5bcbb23
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: e5bcbb23be66d12e428d817fd8b0b495793a939c
Parents: f6a79d0
Author: Todd Nine <to...@apache.org>
Authored: Tue Sep 30 15:47:54 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Tue Sep 30 15:47:54 2014 -0600
----------------------------------------------------------------------
.../mq/cassandra/io/ConsumerTransaction.java | 15 +++++++++++++--
1 file changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e5bcbb23/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
index 6d70924..7205dbc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/ConsumerTransaction.java
@@ -208,9 +208,9 @@ public class ConsumerTransaction extends NoTransactionSearch
long startTime = System.currentTimeMillis();
- UUID startTimeUUID = UUIDUtils.newTimeUUID( startTime, 0 );
+ UUID startTimeUUID = UUIDUtils.newTimeUUID( startTime, 0 ); //this exact moment in time + clockseq + node
- QueueBounds bounds = getQueueBounds( queueId );
+ QueueBounds bounds = getQueueBounds( queueId ); //first write in time, most write in time
//queue has never been written to
if ( bounds == null )
@@ -224,6 +224,14 @@ public class ConsumerTransaction extends NoTransactionSearch
SearchParam params = getParams( queueId, consumerId, query );
+ //if startId is greater than our max, we disregard it and reset to now because we've advanced beyond
+ //"now"
+ if( params.startId != null && UUIDUtils.compare( params.startId, startTimeUUID ) > 0){
+ logger.warn( "Our cursor has advanced beyond the end of the queue due to transactions. Was {}, resetting to {}", params.startId, startTimeUUID );
+ params = new SearchParam( startTimeUUID, params.reversed, false, params.limit );
+ }
+
+
List<UUID> ids = getQueueRange( queueId, bounds, params );
// get a list of ids from the consumer.
@@ -285,6 +293,9 @@ public class ConsumerTransaction extends NoTransactionSearch
// last read messages uuid, whichever is greater
UUID lastReadId = UUIDUtils.max( lastReadTransactionPointer, lastId );
+ //we can only store the min of the queue Id, beyond that we'll cause errors
+ lastReadId = UUIDUtils.min( lastReadId, bounds.getNewest() );
+
writeClientPointer( queueId, consumerId, lastReadId );
}
catch ( UGLockException e )
[03/35] git commit: removing unnecessary updates
Posted by sn...@apache.org.
removing unnecessary updates
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/05f56db1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/05f56db1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/05f56db1
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 05f56db1277c82d1032840bf5a98aaffc9febcfd
Parents: 05caf49
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Sep 26 13:29:17 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Sep 26 13:29:17 2014 -0600
----------------------------------------------------------------------
.../notifications/ApplicationQueueManager.java | 4 ++-
.../services/notifications/JobScheduler.java | 1 +
.../notifications/NotificationsService.java | 7 ++---
.../notifications/NotificationsTaskManager.java | 33 --------------------
.../notifications/SingleQueueTaskManager.java | 2 +-
.../services/notifications/TaskTracker.java | 4 +--
6 files changed, 9 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05f56db1/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
index 766cb75..4bd0cde 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
@@ -91,6 +91,9 @@ public class ApplicationQueueManager implements QueueManager {
}
public void queueNotification(final Notification notification, final JobExecution jobExecution) throws Exception {
+ if(scheduleQueueJob(notification)){
+ return;
+ }
final Meter queueMeter = metricsFactory.getMeter(ApplicationQueueManager.class,"queue");
long startTime = System.currentTimeMillis();
@@ -171,7 +174,6 @@ public class ApplicationQueueManager implements QueueManager {
// update queued time
now = System.currentTimeMillis();
notification.setQueued(System.currentTimeMillis());
- em.update(notification);
LOG.info("ApplicationQueueMessage: notification {} device {} queue time set. duration "+(System.currentTimeMillis()-now)+" ms", notification.getUuid(), deviceRef.getUuid());
}
now = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05f56db1/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java
index 6015a0a..a9acd95 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/JobScheduler.java
@@ -74,6 +74,7 @@ public class JobScheduler{
SchedulerService scheduler = getSchedulerService();
scheduler.createJob("queueJob", scheduleAt, jobData);
LOG.info("notification {} scheduled for queuing", notification.getUuid());
+ em.update(notification);
}
return scheduled;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05f56db1/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
index f45750c..4e5692e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
@@ -143,15 +143,12 @@ public class NotificationsService extends AbstractCollectionService {
Map<String, Object> properties = new HashMap<String, Object>(2);
properties.put("started", notification.getStarted());
properties.put("state", notification.getState());
- em.updateProperties(notification, properties);
+ notification.addProperties(properties);
LOG.info("ApplicationQueueMessage: notification {} properties updated in duration {} ms", notification.getUuid(),System.currentTimeMillis() - now);
}
long now = System.currentTimeMillis();
- if(!notificationQueueManager.scheduleQueueJob(notification)){
- notificationQueueManager.queueNotification(notification, null);
- }
-
+ notificationQueueManager.queueNotification(notification, null);
LOG.info("NotificationService: notification {} post queue duration {} ms ", notification.getUuid(),System.currentTimeMillis() - now);
// future: somehow return 202?
return results;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05f56db1/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsTaskManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsTaskManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsTaskManager.java
deleted file mode 100644
index 0869f73..0000000
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsTaskManager.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.usergrid.services.notifications;
-
-import org.apache.usergrid.persistence.entities.Notifier;
-import org.apache.usergrid.persistence.entities.Receipt;
-
-import java.util.UUID;
-
-public interface NotificationsTaskManager {
-
- void completed(Notifier notifier, Receipt receipt, UUID deviceUUID,
- String newProviderId) throws Exception;
-
- void failed(Notifier notifier, Receipt receipt, UUID deviceUUID, Object code,
- String message) throws Exception;
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05f56db1/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java
index f87f497..e89267d 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/SingleQueueTaskManager.java
@@ -30,7 +30,7 @@ import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
-public class SingleQueueTaskManager implements NotificationsTaskManager {
+public class SingleQueueTaskManager {
private static final Logger LOG = LoggerFactory
.getLogger(SingleQueueTaskManager.class);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05f56db1/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskTracker.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskTracker.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskTracker.java
index 322c4bf..8dfa0dc 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskTracker.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/TaskTracker.java
@@ -26,11 +26,11 @@ import java.util.UUID;
public class TaskTracker {
private Notifier notifier;
- private NotificationsTaskManager taskManager;
+ private SingleQueueTaskManager taskManager;
private Receipt receipt;
private UUID deviceId;
- public TaskTracker(Notifier notifier, NotificationsTaskManager taskManager, Receipt receipt, UUID deviceId) {
+ public TaskTracker(Notifier notifier, SingleQueueTaskManager taskManager, Receipt receipt, UUID deviceId) {
this.notifier = notifier;
this.taskManager = taskManager;
this.receipt = receipt;
[10/35] git commit: Switching back to using ecm.write() instead of
ecm.update() because using update() breaks the Core module tests.
Posted by sn...@apache.org.
Switching back to using ecm.write() instead of ecm.update() because using update() breaks the Core module tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/16cc4dff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/16cc4dff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/16cc4dff
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 16cc4dffd0ca59053e2363927459e8cc838fa9b4
Parents: 38eb9e4
Author: Dave Johnson <dm...@apigee.com>
Authored: Sun Sep 28 11:44:16 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sun Sep 28 11:44:16 2014 -0400
----------------------------------------------------------------------
.../java/org/apache/usergrid/corepersistence/CpEntityManager.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/16cc4dff/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5a5e6bc..f85ba30 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -503,7 +503,8 @@ public class CpEntityManager implements EntityManager {
logger.debug("About to Write {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
- cpEntity = ecm.update( cpEntity ).toBlockingObservable().last();
+ // using ecm.update() here causes Core tests to fail
+ cpEntity = ecm.write( cpEntity ).toBlockingObservable().last();
logger.debug("Wrote {}:{} version {}", new Object[] {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion() });
[23/35] git commit: Fix for BasicIT failure.
Posted by sn...@apache.org.
Fix for BasicIT failure.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4f68f720
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4f68f720
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4f68f720
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 4f68f7207bc3827e817f90fc2184dc649200d7a8
Parents: 009a184
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Sep 29 14:10:11 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Sep 29 14:10:11 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 6 ++--
.../persistence/index/impl/EsQueryVistor.java | 3 +-
.../java/org/apache/usergrid/rest/BasicIT.java | 34 ++++++++++++++------
3 files changed, 31 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4f68f720/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 0345e23..3486fbc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -1627,6 +1627,8 @@ public class CpRelationManager implements RelationManager {
private Results buildResults(Query query, CandidateResults crs, String collName ) {
+ logger.debug("buildResults() for {} from {} candidates", collName, crs.size());
+
Results results = null;
if ( query.getLevel().equals( Level.IDS )) {
@@ -1700,7 +1702,7 @@ public class CpRelationManager implements RelationManager {
logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
new Object[] { cr.getId().getUuid(), cr.getId().getType(),
cr.getVersion(), e.getVersion()});
- continue;
+ continue;
}
org.apache.usergrid.persistence.model.entity.Entity alreadySeen =
@@ -1741,7 +1743,7 @@ public class CpRelationManager implements RelationManager {
results.setCursor( crs.getCursor() );
results.setQueryProcessor( new CpQueryProcessor(em, query, headEntity, collName) );
- logger.debug("Returning results size {}", results.getIds().size() );
+ logger.debug("Returning results size {}", results.size() );
return results;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4f68f720/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 4f9cd60..5634183 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -214,6 +214,7 @@ public class EsQueryVistor implements QueryVisitor {
String svalue = (String)value;
BoolQueryBuilder qb = QueryBuilders.boolQuery(); // let's do a boolean OR
+ qb.minimumNumberShouldMatch(1);
// field is an entity/array that does not need a prefix on its name
qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) );
@@ -264,7 +265,6 @@ public class EsQueryVistor implements QueryVisitor {
// logic to deal with nested property names
// only add prefix to last name in property
-
String[] parts = origname.split("\\.");
if ( parts.length > 1 ) {
name = parts[ parts.length - 1 ];
@@ -286,6 +286,7 @@ public class EsQueryVistor implements QueryVisitor {
name = addStringPrefix( name );
}
+ // re-create nested property name
if ( parts.length > 1 ) {
parts[parts.length - 1] = name;
Joiner joiner = Joiner.on(".").skipNulls();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4f68f720/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java
index ec30753..473052d 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/BasicIT.java
@@ -230,7 +230,8 @@ public class BasicIT extends AbstractRestIT {
err_thrown = false;
try {
- node = mapper.readTree( resource().path( "/test-organization/test-app/users" ).accept( MediaType.APPLICATION_JSON )
+ node = mapper.readTree( resource().path( "/test-organization/test-app/users" )
+ .accept( MediaType.APPLICATION_JSON )
.get( String.class ));
}
catch ( UniformInterfaceException e ) {
@@ -241,8 +242,10 @@ public class BasicIT extends AbstractRestIT {
// test login app user with pin
- node = mapper.readTree( resource().path( "/test-organization/test-app/token" ).queryParam( "grant_type", "pin" )
- .queryParam( "username", "ed@anuff.com" ).queryParam( "pin", "1234" )
+ node = mapper.readTree( resource().path( "/test-organization/test-app/token" )
+ .queryParam( "grant_type", "pin" )
+ .queryParam( "username", "ed@anuff.com" )
+ .queryParam( "pin", "1234" )
.accept( MediaType.APPLICATION_JSON ).get( String.class ));
logNode( node );
@@ -254,22 +257,35 @@ public class BasicIT extends AbstractRestIT {
MultivaluedMap<String, String> formData = new MultivaluedMapImpl();
formData.add( "pin", "5678" );
- node = mapper.readTree( resource().path( "/test-organization/test-app/users/ed@anuff.com/setpin" )
- .queryParam( "access_token", user_access_token ).type( "application/x-www-form-urlencoded" )
+ node = mapper.readTree( resource()
+ .path( "/test-organization/test-app/users/ed@anuff.com/setpin" )
+ .queryParam( "access_token", user_access_token )
+ .type( "application/x-www-form-urlencoded" )
.post( String.class, formData ));
- node = mapper.readTree( resource().path( "/test-organization/test-app/token" ).queryParam( "grant_type", "pin" )
- .queryParam( "username", "ed@anuff.com" ).queryParam( "pin", "5678" )
- .accept( MediaType.APPLICATION_JSON ).get( String.class ));
+ refreshIndex("test-organization", "test-app");
+
+ node = mapper.readTree( resource()
+ .path( "/test-organization/test-app/token" )
+ .queryParam( "grant_type", "pin" )
+ .queryParam( "username", "ed@anuff.com" )
+ .queryParam( "pin", "5678" )
+ .accept( MediaType.APPLICATION_JSON )
+ .get( String.class ));
logNode( node );
user_access_token = node.get( "access_token" ).textValue();
assertTrue( isNotBlank( user_access_token ) );
+ refreshIndex("test-organization", "test-app");
+
// test user test extension resource
- node = mapper.readTree( resource().path( "/test-organization/test-app/users/ed@anuff.com/test" ).get( String.class ));
+ node = mapper.readTree( resource()
+ .path( "/test-organization/test-app/users/ed@anuff.com/test" )
+ .queryParam( "access_token", user_access_token )
+ .get( String.class ));
logNode( node );
// test create user with guest permissions (no token)
[12/35] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3f6d7b4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3f6d7b4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3f6d7b4c
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 3f6d7b4c8ce3d2972a28574f0f3ce4bcc5542890
Parents: 68ee726 86edd64
Author: Dave Johnson <dm...@apigee.com>
Authored: Sun Sep 28 11:45:08 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sun Sep 28 11:45:08 2014 -0400
----------------------------------------------------------------------
.../usergrid/services/notifications/ApplicationQueueManager.java | 2 +-
.../apache/usergrid/services/notifications/QueueListener.java | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[16/35] git commit: Fix bug that was preventing boolean fields from
being indexed correctly in ES.
Posted by sn...@apache.org.
Fix bug that was preventing boolean fields from being indexed correctly in ES.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/49108ef6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/49108ef6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/49108ef6
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 49108ef6c2134d9e8c46c0be3e163df1dfefd907
Parents: 3f6d7b4
Author: Dave Johnson <dm...@apigee.com>
Authored: Sun Sep 28 12:36:49 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sun Sep 28 12:36:49 2014 -0400
----------------------------------------------------------------------
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/49108ef6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 4843854..15fadd5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -528,10 +528,6 @@ public class EsEntityIndexImpl implements EntityIndex {
locMap.put("lon", locField.getValue().getLongtitude());
entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap);
- } else if ( f instanceof BooleanField ) {
-
- entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue());
-
} else if ( f instanceof DoubleField
|| f instanceof FloatField
|| f instanceof IntegerField
[02/35] git commit: setup default sleep window
Posted by sn...@apache.org.
setup default sleep window
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/05caf493
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/05caf493
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/05caf493
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 05caf49325e52f806b3fdc22f18b70512f722d8c
Parents: f2a93d7
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Sep 26 11:23:19 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Sep 26 11:23:19 2014 -0600
----------------------------------------------------------------------
.../services/notifications/ApplicationQueueManager.java | 5 +++--
.../apache/usergrid/services/notifications/QueueListener.java | 3 ++-
.../services/notifications/apns/NotificationsServiceIT.java | 4 +++-
.../services/notifications/gcm/NotificationsServiceIT.java | 4 +++-
4 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05caf493/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
index 198d41a..766cb75 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
@@ -47,7 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public class ApplicationQueueManager implements QueueManager {
- public static String DEFAULT_QUEUE_NAME = "notifications/queuelistenerv1_12";
+ public static String DEFAULT_QUEUE_NAME = "notifications/queuelistenerv1_20;notifications/queuelistenerv1_21";
public static final String DEFAULT_QUEUE_PROPERTY = "usergrid.notifications.listener.queueName";
private static final Logger LOG = LoggerFactory.getLogger(ApplicationQueueManager.class);
@@ -429,7 +429,8 @@ public class ApplicationQueueManager implements QueueManager {
public static String getRandomQueue(String[] queueNames) {
int size = queueNames.length;
Random random = new Random();
- return queueNames[random.nextInt(size)];
+ String name = queueNames[random.nextInt(size)];
+ return name;
}
private static final class IteratorObservable<T> implements rx.Observable.OnSubscribe<T> {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05caf493/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index 1d4c5cb..234821b 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -36,6 +36,7 @@ import java.util.concurrent.atomic.AtomicInteger;
public class QueueListener {
public static final long MESSAGE_TRANSACTION_TIMEOUT = 60 * 5 * 1000;
+ public static long DEFAULT_SLEEP = 5000;
private static final Logger LOG = LoggerFactory.getLogger(QueueListener.class);
@@ -91,7 +92,7 @@ public class QueueListener {
try {
sleepBetweenRuns = new Long(properties.getProperty("usergrid.notifications.listener.sleep.between", "0")).longValue();
- sleepWhenNoneFound = new Long(properties.getProperty("usergrid.notifications.listener.sleep.after", "5000")).longValue();
+ sleepWhenNoneFound = new Long(properties.getProperty("usergrid.notifications.listener.sleep.after", ""+DEFAULT_SLEEP)).longValue();
batchSize = new Integer(properties.getProperty("usergrid.notifications.listener.batchSize", (""+batchSize)));
queueNames = ApplicationQueueManager.getQueueNames(properties);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05caf493/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
index ec7a14f..a47dc99 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
@@ -69,6 +69,9 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
@Override
@Before
public void before() throws Exception {
+ ApplicationQueueManager.DEFAULT_QUEUE_NAME = "notifications/test/" + UUID.randomUUID().toString()+";"+"notifications/test/" + UUID.randomUUID().toString();
+
+ QueueListener.DEFAULT_SLEEP = 200;
super.before();
// create apns notifier //
app.clear();
@@ -128,7 +131,6 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
ns.TEST_PATH_QUERY = pathQuery;
- ApplicationQueueManager.DEFAULT_QUEUE_NAME = "notifications/test/" + UUID.randomUUID().toString();
listener = new QueueListener(ns.getServiceManagerFactory(),ns.getEntityManagerFactory(),ns.getMetricsFactory(), new Properties());
listener.run();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/05caf493/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 09814f3..eb3c8b8 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -63,6 +63,9 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
@Override
@Before
public void before() throws Exception {
+ ApplicationQueueManager.DEFAULT_QUEUE_NAME = "notifications/test/" + UUID.randomUUID().toString()+";"+"notifications/test/" + UUID.randomUUID().toString();
+ QueueListener.DEFAULT_SLEEP = 200;
+
super.before();
// create gcm notifier //
@@ -102,7 +105,6 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
PathQuery pathQuery = new PathQuery(new SimpleEntityRef( app.getEm().getApplicationRef()), query);
ns.TEST_PATH_QUERY = pathQuery;
- ApplicationQueueManager.DEFAULT_QUEUE_NAME = "notifications/test/" + UUID.randomUUID().toString();
listener = new QueueListener(ns.getServiceManagerFactory(),
ns.getEntityManagerFactory(),ns.getMetricsFactory(), new Properties());
listener.run();
[18/35] git commit: Fix for array querying regression caused by move
to new ES field naming schema with prefixes instead of suffixes.
Posted by sn...@apache.org.
Fix for array querying regression caused by move to new ES field naming schema with prefixes instead of suffixes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/009a184d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/009a184d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/009a184d
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 009a184df6f7f43e08c6a43fff17e28193821218
Parents: fb050ff
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Sep 29 09:30:22 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Sep 29 09:30:22 2014 -0400
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 29 +++++++++-----
.../persistence/index/impl/EsQueryVistor.java | 42 +++++++++++++-------
2 files changed, 48 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/009a184d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 15fadd5..8a1233f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -18,6 +18,7 @@
package org.apache.usergrid.persistence.index.impl;
+import com.google.common.base.Joiner;
import com.google.common.util.concurrent.AtomicDouble;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -57,10 +58,7 @@ import org.apache.usergrid.persistence.model.field.SetField;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
-import org.elasticsearch.action.ActionFuture;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
-import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
@@ -413,20 +411,20 @@ public class EsEntityIndexImpl implements EntityIndex {
order = SortOrder.DESC;
}
- String stringFieldName = STRING_PREFIX + sp.getPropertyName();
- FieldSortBuilder sort = SortBuilders
+ final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder stringSort = SortBuilders
.fieldSort( stringFieldName )
.order(order)
.ignoreUnmapped(true);
- srb.addSort( sort );
+ srb.addSort( stringSort );
log.debug(" Sort: {} order by {}", stringFieldName, order.toString());
- String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
- sort = SortBuilders
+ final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder numberSort = SortBuilders
.fieldSort( numberFieldName )
.order(order)
.ignoreUnmapped(true);
- srb.addSort( sort );
+ srb.addSort( numberSort );
log.debug(" Sort: {} order by {}", numberFieldName, order.toString());
}
@@ -490,6 +488,7 @@ public class EsEntityIndexImpl implements EntityIndex {
Map<String, Object> entityMap = new HashMap<String, Object>();
for (Object f : entity.getFields().toArray()) {
+
Field field = (Field) f;
if (f instanceof ListField) {
@@ -497,6 +496,16 @@ public class EsEntityIndexImpl implements EntityIndex {
entityMap.put(field.getName().toLowerCase(),
new ArrayList(processCollectionForMap(list)));
+ if ( !list.isEmpty() ) {
+ if ( list.get(0) instanceof String ) {
+ Joiner joiner = Joiner.on(" ").skipNulls();
+ String joined = joiner.join(list);
+ entityMap.put( ANALYZED_STRING_PREFIX + field.getName().toLowerCase(),
+ new ArrayList(processCollectionForMap(list)));
+
+ }
+ }
+
} else if (f instanceof ArrayField) {
List list = (List) field.getValue();
entityMap.put(field.getName().toLowerCase(),
@@ -511,6 +520,8 @@ public class EsEntityIndexImpl implements EntityIndex {
EntityObject eo = (EntityObject)field.getValue();
entityMap.put(field.getName().toLowerCase(), entityToMap(eo)); // recursion
+ // Add type information as field-name prefixes
+
} else if (f instanceof StringField) {
// index in lower case because Usergrid queries are case insensitive
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/009a184d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 07f98a2..4f9cd60 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -150,8 +150,17 @@ public class EsQueryVistor implements QueryVisitor {
String name = op.getProperty().getValue();
name = name.toLowerCase();
Object value = op.getLiteral().getValue();
- name = addPrefix( value, name, true );
- stack.push( QueryBuilders.matchQuery( name, value ));
+
+ BoolQueryBuilder qb = QueryBuilders.boolQuery(); // let's do a boolean OR
+ qb.minimumNumberShouldMatch(1);
+
+ // field is an entity/array that needs no name prefix
+ qb = qb.should( QueryBuilders.matchQuery( name, value ) );
+
+ // OR field is a string and needs the prefix on the name
+ qb = qb.should( QueryBuilders.matchQuery( addPrefix( value.toString(), name, true), value));
+
+ stack.push( qb );
}
@@ -203,19 +212,24 @@ public class EsQueryVistor implements QueryVisitor {
if ( value instanceof String ) {
String svalue = (String)value;
-
- // use normal prefix, we need unanalyzed field for equals op
- name = addPrefix( value, name );
+ BoolQueryBuilder qb = QueryBuilders.boolQuery(); // let's do a boolean OR
+
+ // field is an entity/array that does not need a prefix on its name
+ qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) );
+
+ // or field is just a string that does need a prefix
if ( svalue.indexOf("*") != -1 ) {
- stack.push( QueryBuilders.wildcardQuery(name, svalue) );
- return;
+ qb = qb.should( QueryBuilders.wildcardQuery( addPrefix( value, name ), svalue ) );
+ } else {
+ qb = qb.should( QueryBuilders.termQuery( addPrefix( value, name ), value ));
}
+ stack.push( qb );
+ return;
+ }
- } else {
- name = addPrefix( value, name );
- }
- stack.push( QueryBuilders.termQuery( name, value ));
+ // assume all other types need prefix
+ stack.push( QueryBuilders.termQuery( addPrefix( value, name ), value ));
}
@@ -263,13 +277,13 @@ public class EsQueryVistor implements QueryVisitor {
name = addStringPrefix( name );
} else if ( value instanceof Number ) {
- name = addNumberPrefix(name);
+ name = addNumberPrefix( name );
} else if ( value instanceof Boolean ) {
- name = addBooleanPrefix(name);
+ name = addBooleanPrefix( name );
} else if ( value instanceof UUID ) {
- name = addStringPrefix(name);
+ name = addStringPrefix( name );
}
if ( parts.length > 1 ) {
[31/35] git commit: Added additional logging and a short circuit test
Posted by sn...@apache.org.
Added additional logging and a short circuit test
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/87c51ae5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/87c51ae5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/87c51ae5
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 87c51ae581aa5f676af0e2f2dcbd876cf8a16da1
Parents: 9ce8346
Author: Todd Nine <to...@apache.org>
Authored: Tue Sep 30 14:03:12 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Tue Sep 30 14:03:12 2014 -0600
----------------------------------------------------------------------
.../mq/cassandra/io/AbstractSearch.java | 42 +++++++++++++++-----
1 file changed, 32 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/87c51ae5/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
index b0b5ac7..ffda843 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
@@ -37,6 +37,8 @@ import org.apache.usergrid.persistence.exceptions.QueueException;
import org.apache.usergrid.persistence.hector.CountingMutator;
import org.apache.usergrid.utils.UUIDUtils;
+import com.fasterxml.uuid.UUIDComparator;
+
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
import me.prettyprint.hector.api.beans.HColumn;
@@ -63,6 +65,7 @@ import static org.apache.usergrid.persistence.cassandra.Serializers.be;
import static org.apache.usergrid.persistence.cassandra.Serializers.se;
import static org.apache.usergrid.persistence.cassandra.Serializers.ue;
import static org.apache.usergrid.utils.NumberUtils.roundLong;
+import static org.apache.usergrid.utils.UUIDUtils.compare;
import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMillis;
@@ -181,25 +184,26 @@ public abstract class AbstractSearch implements QueueSearch {
current_ts_shard = finish_ts_shard;
}
+ final MessageIdComparator comparator = new MessageIdComparator(params.reversed);
+
//should be start < finish
- if ( !params.reversed && UUIDUtils.compare( start, finish_uuid ) > 0 ) {
+ if ( comparator.compare( start, finish_uuid ) > 0 ) {
logger.warn( "Tried to perform a slice with start UUID {} after finish UUID {}.", start, finish_uuid );
- throw new IllegalArgumentException( String.format("You cannot specify a start value of %s after finish value of %s", start, finish_uuid) );
+ throw new IllegalArgumentException(
+ String.format( "You cannot specify a start value of %s after finish value of %s", start,
+ finish_uuid ) );
}
- // should be finish < start
- else if ( params.reversed && UUIDUtils.compare( start, finish_uuid ) < 0 ) {
- logger.warn( "Tried to perform a slice with start UUID {} after finish UUID {}.", start, finish_uuid );
- throw new IllegalArgumentException( String.format("You cannot specify a start value of %s after finish value of %s", start, finish_uuid) );
- }
UUID lastValue = start;
boolean firstPage = true;
- while ( ( current_ts_shard >= start_ts_shard ) && ( current_ts_shard <= finish_ts_shard ) ) {
+ while ( ( current_ts_shard >= start_ts_shard ) && ( current_ts_shard <= finish_ts_shard ) && comparator.compare(start, finish_uuid) < 1 ) {
+
+ logger.info( "Starting search with start UUID {}, finish UUID {}, and reversed {}", new Object[]{lastValue, finish_uuid, params.reversed });
SliceQuery<ByteBuffer, UUID, ByteBuffer> q = createSliceQuery( ko, be, ue, be );
@@ -215,8 +219,8 @@ public abstract class AbstractSearch implements QueueSearch {
final UUID columnName = column.getName();
// skip the first one, we've already read it
- if ( i == 0 && (firstPage && params.skipFirst && params.startId.equals( columnName ))
- || (!firstPage && lastValue != null && lastValue.equals(columnName)) ) {
+ if ( i == 0 && ( firstPage && params.skipFirst && params.startId.equals( columnName ) ) || ( !firstPage
+ && lastValue != null && lastValue.equals( columnName ) ) ) {
continue;
}
@@ -333,4 +337,22 @@ public abstract class AbstractSearch implements QueueSearch {
return o1Idx - o2Idx;
}
}
+
+
+ private static final class MessageIdComparator implements Comparator<UUID> {
+
+ private final int comparator;
+
+
+ private MessageIdComparator( final boolean reversed ) {
+
+ this.comparator = reversed ? -1 : 1;
+ }
+
+
+ @Override
+ public int compare( final UUID o1, final UUID o2 ) {
+ return UUIDUtils.compare( o1, o2 )*comparator;
+ }
+ }
}
[13/35] git commit: removing static from tests
Posted by sn...@apache.org.
removing static from tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c0f377df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c0f377df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c0f377df
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: c0f377dfdc03ea25cb3452c3a9bad3e290e12746
Parents: 86edd64
Author: Shawn Feldman <sf...@apache.org>
Authored: Sun Sep 28 09:49:40 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Sun Sep 28 09:49:40 2014 -0600
----------------------------------------------------------------------
.../apache/usergrid/services/notifications/QueueListener.java | 6 +++---
.../services/notifications/apns/NotificationsServiceIT.java | 2 +-
.../services/notifications/gcm/NotificationsServiceIT.java | 3 ++-
3 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0f377df/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index 056c3ca..043bcb2 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -35,8 +35,8 @@ import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
public class QueueListener {
- public static final long MESSAGE_TRANSACTION_TIMEOUT = 60 * 5 * 1000;
- public static long DEFAULT_SLEEP = 5000;
+ public final long MESSAGE_TRANSACTION_TIMEOUT = 60 * 5 * 1000;
+ public long DEFAULT_SLEEP = 5000;
private static final Logger LOG = LoggerFactory.getLogger(QueueListener.class);
@@ -63,7 +63,7 @@ public class QueueListener {
ExecutorService pool;
List<Future> futures;
- public static final String MAX_THREADS = "2";
+ public final String MAX_THREADS = "2";
private Integer batchSize = 1000;
private String[] queueNames;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0f377df/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
index a47dc99..1985b19 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
@@ -71,7 +71,6 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
public void before() throws Exception {
ApplicationQueueManager.DEFAULT_QUEUE_NAME = "notifications/test/" + UUID.randomUUID().toString()+";"+"notifications/test/" + UUID.randomUUID().toString();
- QueueListener.DEFAULT_SLEEP = 200;
super.before();
// create apns notifier //
app.clear();
@@ -132,6 +131,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
ns.TEST_PATH_QUERY = pathQuery;
listener = new QueueListener(ns.getServiceManagerFactory(),ns.getEntityManagerFactory(),ns.getMetricsFactory(), new Properties());
+ listener.DEFAULT_SLEEP = 200;
listener.run();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c0f377df/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index eb3c8b8..fdcf7b6 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -64,7 +64,6 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
@Before
public void before() throws Exception {
ApplicationQueueManager.DEFAULT_QUEUE_NAME = "notifications/test/" + UUID.randomUUID().toString()+";"+"notifications/test/" + UUID.randomUUID().toString();
- QueueListener.DEFAULT_SLEEP = 200;
super.before();
@@ -107,6 +106,8 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
ns.TEST_PATH_QUERY = pathQuery;
listener = new QueueListener(ns.getServiceManagerFactory(),
ns.getEntityManagerFactory(),ns.getMetricsFactory(), new Properties());
+ listener.DEFAULT_SLEEP = 200;
+
listener.run();
}
[21/35] git commit: Fixes bug with invalid scan range being build on
shard scan
Posted by sn...@apache.org.
Fixes bug with invalid scan range being build on shard scan
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cbea83cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cbea83cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cbea83cd
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: cbea83cd4d38935ae3f5342c97018cb0ea41e2fb
Parents: deef46b
Author: Todd Nine <to...@apache.org>
Authored: Mon Sep 29 12:01:38 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Mon Sep 29 12:01:38 2014 -0600
----------------------------------------------------------------------
.../usergrid/mq/cassandra/io/AbstractSearch.java | 17 +++--------------
1 file changed, 3 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cbea83cd/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
index 48b7d3d..c3d2800 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
@@ -194,26 +194,15 @@ public abstract class AbstractSearch implements QueueSearch
current_ts_shard = finish_ts_shard;
}
- while ( ( current_ts_shard >= start_ts_shard ) && ( current_ts_shard <= finish_ts_shard ) )
- {
-
- UUID slice_start = MIN_TIME_UUID;
- UUID slice_end = MAX_TIME_UUID;
- if ( current_ts_shard == start_ts_shard )
- {
- slice_start = start;
- }
- if ( current_ts_shard == finish_ts_shard )
- {
- slice_end = finish_uuid;
- }
+ while ( ( current_ts_shard >= start_ts_shard ) && ( current_ts_shard <= finish_ts_shard ) )
+ {
SliceQuery<ByteBuffer, UUID, ByteBuffer> q = createSliceQuery( ko, be, ue, be );
q.setColumnFamily( QUEUE_INBOX.getColumnFamily() );
q.setKey( getQueueShardRowKey( queueId, current_ts_shard ) );
- q.setRange( slice_start, slice_end, params.reversed, params.limit + 1 );
+ q.setRange( start, finish_uuid, params.reversed, params.limit + 1 );
List<HColumn<UUID, ByteBuffer>> cassResults = q.execute().get().getColumns();
[08/35] git commit: fix get notification receipts again
Posted by sn...@apache.org.
fix get notification receipts again
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/38eb9e4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/38eb9e4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/38eb9e4b
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 38eb9e4b8acd32380f3c0f3ea5923498a5b5f010
Parents: a200f45
Author: Shawn Feldman <sf...@apache.org>
Authored: Sat Sep 27 10:42:58 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Sat Sep 27 10:42:58 2014 -0600
----------------------------------------------------------------------
portal/js/global/ug-service.js | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/38eb9e4b/portal/js/global/ug-service.js
----------------------------------------------------------------------
diff --git a/portal/js/global/ug-service.js b/portal/js/global/ug-service.js
index db2bfe9..0f671ff 100644
--- a/portal/js/global/ug-service.js
+++ b/portal/js/global/ug-service.js
@@ -551,7 +551,8 @@ AppServices.Services.factory('ug', function(configuration, $rootScope, utility,
});
},
getNotificationReceipts: function(uuid) {
- this.getCollection('receipts', 'notifications/' + uuid + '/receipts');
+ this.getCollection('receipts', 'receipts', 'created desc',
+ 'notificationUUID=' + uuid);
var self = this;
$rootScope.$on('receipts-received', function(event, receipts) {
self.receiptsCollection = receipts;
@@ -1369,4 +1370,4 @@ AppServices.Services.factory('ug', function(configuration, $rootScope, utility,
}
}
-});
+});
\ No newline at end of file
[26/35] git commit: shorter transaction times; smaller batches
Posted by sn...@apache.org.
shorter transaction times; smaller batches
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/48a3af16
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/48a3af16
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/48a3af16
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 48a3af163cbf120f55adab1d16bb91edaba1af66
Parents: 2ea3ee1
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Sep 29 16:07:54 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Sep 29 16:07:54 2014 -0600
----------------------------------------------------------------------
.../apache/usergrid/services/notifications/QueueListener.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/48a3af16/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index 42f9dc4..31df5e7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -35,12 +35,12 @@ import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
public class QueueListener {
- public final long MESSAGE_TRANSACTION_TIMEOUT = 60 * 5 * 1000;
+ public final long MESSAGE_TRANSACTION_TIMEOUT = 1 * 60 * 1000;
+
public long DEFAULT_SLEEP = 5000;
private static final Logger LOG = LoggerFactory.getLogger(QueueListener.class);
-
private MetricsFactory metricsService;
private ServiceManagerFactory smf;
@@ -62,7 +62,7 @@ public class QueueListener {
private List<Future> futures;
public final String MAX_THREADS = "2";
- private Integer batchSize = 1000;
+ private Integer batchSize = 100;
private String[] queueNames;
[07/35] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Posted by sn...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a200f45d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a200f45d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a200f45d
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: a200f45db15fd45be7bf2ef49b3bba12d5b051c3
Parents: 43b85a6 0dc230d
Author: Dave Johnson <dm...@apigee.com>
Authored: Sat Sep 27 10:56:15 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Sat Sep 27 10:56:15 2014 -0400
----------------------------------------------------------------------
.../notifications/ApplicationQueueManager.java | 38 +++++++++++---------
.../notifications/NotificationsService.java | 7 ++--
.../notifications/NotificationsTaskManager.java | 33 -----------------
.../services/notifications/QueueListener.java | 20 +++++------
.../services/notifications/QueueManager.java | 4 ---
.../notifications/SingleQueueTaskManager.java | 10 +++---
.../services/notifications/TaskTracker.java | 4 +--
.../apns/NotificationsServiceIT.java | 4 ++-
.../gcm/NotificationsServiceIT.java | 4 ++-
9 files changed, 47 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
[05/35] git commit: default threads to two; less logging; more queues
Posted by sn...@apache.org.
default threads to two; less logging; more queues
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0dc230d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0dc230d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0dc230d3
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 0dc230d3db278a3ec5dab2630862db0af6351e5c
Parents: 96ea029
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Sep 26 17:05:02 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Sep 26 17:05:02 2014 -0600
----------------------------------------------------------------------
.../usergrid/services/notifications/ApplicationQueueManager.java | 4 +---
.../apache/usergrid/services/notifications/QueueListener.java | 2 +-
2 files changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0dc230d3/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
index d9542ed..dac3b6a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/ApplicationQueueManager.java
@@ -47,7 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public class ApplicationQueueManager implements QueueManager {
- public static String DEFAULT_QUEUE_NAME = "notifications/queuelistenerv1_20;notifications/queuelistenerv1_21";
+ public static String DEFAULT_QUEUE_NAME = "notifications/queuelistenerv1_20;notifications/queuelistenerv1_21;notifications/queuelistenerv1_22";
public static final String DEFAULT_QUEUE_PROPERTY = "usergrid.notifications.listener.queueName";
private static final Logger LOG = LoggerFactory.getLogger(ApplicationQueueManager.class);
@@ -242,8 +242,6 @@ public class ApplicationQueueManager implements QueueManager {
if (LOG.isInfoEnabled()) {
long elapsed = notification.getQueued() != null ? notification.getQueued() - startTime : 0;
LOG.info("ApplicationQueueMessage: notification {} done queuing to {} devices in "+elapsed+" ms",notification.getUuid().toString(),deviceCount.get());
- LOG.info("ApplicationQueueMessage: notification {} finished in {} ms",notification.getUuid().toString(),elapsed);
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0dc230d3/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
index 234821b..6281ef9 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
@@ -63,7 +63,7 @@ public class QueueListener {
ExecutorService pool;
List<Future> futures;
- public static final String MAX_THREADS = "1";
+ public static final String MAX_THREADS = "2";
private Integer batchSize = 1000;
private String[] queueNames;