You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/10/15 16:17:51 UTC
[01/50] [abbrv] usergrid git commit: remove condition
Repository: usergrid
Updated Branches:
refs/heads/asf-site 60acf84a1 -> ec2c44788
refs/heads/usergrid-1007-shiro-cache 542763185 -> 341b47365
remove condition
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b437f618
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b437f618
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b437f618
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: b437f618775180a7b4a4a8e174ea7f152c4aa4fc
Parents: f35d01c
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 5 14:24:57 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 5 14:24:57 2015 -0600
----------------------------------------------------------------------
.../usergrid/persistence/queue/impl/QueueManagerFactoryImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b437f618/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueManagerFactoryImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueManagerFactoryImpl.java
index a1940d0..d0ed1ef 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueManagerFactoryImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QueueManagerFactoryImpl.java
@@ -47,7 +47,7 @@ public class QueueManagerFactoryImpl implements QueueManagerFactory {
}
@Override
public QueueManager getQueueManager(QueueScope scope) {
- if(true==false){
+ if(queueFig.overrideQueueForDefault()){
QueueManager manager = defaultManager.get(scope.getName());
if(manager==null){
manager = new DefaultQueueManager();
[05/50] [abbrv] usergrid git commit: fix observable class cast
exception
Posted by sn...@apache.org.
fix observable class cast exception
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d6bf2fac
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d6bf2fac
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d6bf2fac
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: d6bf2fac1b5b7ee994d13c7efce007d1f0dc6de2
Parents: 1a1d42e
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 5 18:36:05 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 5 18:36:05 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 37 +++++++++++---------
1 file changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d6bf2fac/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index bf29c5a..abb76c2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -233,8 +233,12 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
Observable<IndexEventResult> masterObservable = Observable.from(messages).flatMap(message -> {
- final AsyncEvent event = (AsyncEvent) message.getBody();
-
+ AsyncEvent event = null;
+ try{
+ event = (AsyncEvent) message.getBody();
+ }catch (ClassCastException cce){
+ logger.error("Failed to deserialize message body",cce);
+ }
logger.debug("Processing {} event", event);
if (event == null) {
@@ -259,10 +263,10 @@ public class AmazonAsyncEventService implements AsyncEventService {
logger.error("Unknown EventType: {}", event);
return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), true));
}
- }catch (Exception e){
- logger.error("Failed to index entity", e,message);
+ } catch (Exception e) {
+ logger.error("Failed to index entity", e, message);
return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), false));
- }finally {
+ } finally {
messageCycle.update(System.currentTimeMillis() - event.getCreationTime());
}
@@ -270,25 +274,25 @@ public class AmazonAsyncEventService implements AsyncEventService {
return masterObservable
//remove unsuccessful
- .filter( indexEventResult -> indexEventResult.shouldProcess() )
+ .filter(indexEventResult -> indexEventResult.shouldProcess())
//take the max
.buffer( MAX_TAKE )
//map them to index results and return them
- .flatMap( indexEventResults -> {
+ .flatMap(indexEventResults -> {
IndexOperationMessage combined = new IndexOperationMessage();
indexEventResults.stream().forEach(
- indexEventResult ->{
- if(indexEventResult.getIndexOperationMessage().isPresent()) {
+ indexEventResult -> {
+ if (indexEventResult.getIndexOperationMessage().isPresent()) {
combined.ingest(indexEventResult.getIndexOperationMessage().get());
}
- } );
+ });
//ack after successful completion of the operation.
return indexProducer.put(combined)
.flatMap(operationResult -> Observable.from(indexEventResults));
- } );
+ });
}
@@ -376,8 +380,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
- final Observable<IndexOperationMessage> edgeIndexObservable = ecm.load(edgeIndexEvent.getEntityId()).flatMap( entity -> eventBuilder.buildNewEdge(
- applicationScope, entity, edge ) );
+ final Observable<IndexOperationMessage> edgeIndexObservable = ecm.load(edgeIndexEvent.getEntityId()).flatMap(entity -> eventBuilder.buildNewEdge(
+ applicationScope, entity, edge));
return edgeIndexObservable;
}
@@ -442,8 +446,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
entityDeleteResults = eventBuilder.buildEntityDelete( applicationScope, entityId );
- final Observable merged = Observable.merge( entityDeleteResults.getEntitiesCompacted(),
- entityDeleteResults.getIndexObservable() );
+ final Observable<IndexOperationMessage> merged = entityDeleteResults.getEntitiesCompacted().flatMap(mvccLogEntries -> entityDeleteResults.getIndexObservable()) ;
return merged;
}
@@ -537,12 +540,12 @@ public class AmazonAsyncEventService implements AsyncEventService {
.map(messages ->
handleMessages(messages)
.map(indexEventResult -> {
- ack( indexEventResult.getQueueMessage() );
+ ack(indexEventResult.getQueueMessage());
return indexEventResult;
})
.toBlocking().lastOrDefault(null)
)//ack each message, but only if we didn't error. If we did, we'll want to log it and
- .subscribeOn( Schedulers.newThread() );
+ .subscribeOn(Schedulers.newThread());
//start in the background
[30/50] [abbrv] usergrid git commit: remove observables in favor of
streams()
Posted by sn...@apache.org.
remove observables in favor of streams()
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b4855172
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b4855172
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b4855172
Branch: refs/heads/asf-site
Commit: b4855172283a62d178d157be112be345c1e150d8
Parents: 0692faa
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 8 18:07:23 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 8 18:07:23 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 95 +++++++++++---------
1 file changed, 51 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b4855172/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 5a46aed..0ac3860 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
import com.google.common.base.Optional;
import org.apache.usergrid.persistence.index.impl.IndexProducer;
@@ -255,22 +256,21 @@ public class AmazonAsyncEventService implements AsyncEventService {
logger.debug("handleMessages with {} message", messages.size());
}
- final int bufferSize = messages.size();
- Observable<IndexEventResult> masterObservable = Observable.from(messages).flatMap(message -> {
+ Observable<IndexEventResult> masterObservable = Observable.from(messages).map(message -> {
AsyncEvent event = null;
- try{
+ try {
event = (AsyncEvent) message.getBody();
- }catch (ClassCastException cce){
- logger.error("Failed to deserialize message body",cce);
+ } catch (ClassCastException cce) {
+ logger.error("Failed to deserialize message body", cce);
}
if (event == null) {
logger.error("AsyncEvent type or event is null!");
- return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(),System.currentTimeMillis()));
+ return new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(), System.currentTimeMillis());
}
final AsyncEvent thisEvent = event;
- if(logger.isDebugEnabled()) {
+ if (logger.isDebugEnabled()) {
logger.debug("Processing {} event", event);
}
@@ -287,53 +287,55 @@ public class AmazonAsyncEventService implements AsyncEventService {
indexoperationObservable = handleEntityIndexUpdate(message);
} else if (event instanceof InitializeApplicationIndexEvent) {
//does not return observable
- handleInitializeApplicationIndex(event,message);
+ handleInitializeApplicationIndex(event, message);
indexoperationObservable = Observable.just(new IndexOperationMessage());
} else {
throw new Exception("Unknown EventType");//TODO: print json instead
}
+ //collect all of the
+ IndexOperationMessage indexOperationMessage =
+ indexoperationObservable
+ .collect(() -> new IndexOperationMessage(), (collector, single ) -> collector.ingest(single))
+ .toBlocking().lastOrDefault(null);
+
//return type that can be indexed and ack'd later
- return indexoperationObservable
- .map(indexOperationMessage ->
- new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage),thisEvent.getCreationTime())
- );
+ return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
} catch (Exception e) {
logger.error("Failed to index message: " + message.getMessageId(), e, message);
- return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime()));
+ return new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime());
}
});
+ //resolve the list and return it.
+ final List<IndexEventResult> indexEventResults = masterObservable.toList().toBlocking().lastOrDefault(null);
+ //if nothing came back then return null
+ if(indexEventResults==null){
+ return null;
+ }
- //filter for success, send to the index(optional), ack
- return masterObservable
- //take the max
- .buffer(bufferSize)
- //map them to index results and return them
- .flatMap(indexEventResults -> {
- IndexOperationMessage combined = new IndexOperationMessage();
- indexEventResults.stream().forEach(
- indexEventResult -> {
- if (indexEventResult.getIndexOperationMessage().isPresent()) {
- combined.ingest(indexEventResult.getIndexOperationMessage().get());
- }
- });
-
+ final IndexOperationMessage combined = new IndexOperationMessage();
+ //stream and filer the messages
+ List<QueueMessage> messagesToAck = indexEventResults.stream()
+ .map(indexEventResult -> {
+ if (indexEventResult.getIndexOperationMessage().isPresent()) {
+ combined.ingest(indexEventResult.getIndexOperationMessage().get());
+ }
+ return indexEventResult;
+ })
+ .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
+ .map(indexEventResult -> {
+ messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime());
+ return indexEventResult;
+ })
//ack after successful completion of the operation.
- return indexProducer.put(combined)
- //change observable type
- .flatMap(indexOperationMessage -> Observable.from(indexEventResults))
- //remove unsuccessful
- .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
- //measure
- .doOnNext(indexEventResult -> messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()))
- //return the queue messages to ack
- .map(result -> result.getQueueMessage().get())
- .toList();
+ .map(result -> result.getQueueMessage().get())
+ .collect(Collectors.toList());
- })
- .doOnError(t -> logger.error("Failed to process queuemessages",t))
- .toBlocking().lastOrDefault(null);
+ //send the batch
+ indexProducer.put(combined).toBlocking().lastOrDefault(null);
+
+ return messagesToAck;
}
@@ -473,11 +475,11 @@ public class AmazonAsyncEventService implements AsyncEventService {
entityDeleteResults = eventBuilder.buildEntityDelete( applicationScope, entityId );
- final Observable<IndexOperationMessage> merged = entityDeleteResults
+ entityDeleteResults
.getEntitiesCompacted()
- .collect(() -> new ArrayList<>(), (list, item) -> list.add(item))
- .flatMap(collected -> entityDeleteResults.getIndexObservable()) ;
- return merged;
+ .collect(() -> new ArrayList<>(), (list, item) -> list.add(item)).toBlocking().lastOrDefault(null);
+
+ return entityDeleteResults.getIndexObservable();
}
@@ -574,10 +576,15 @@ public class AmazonAsyncEventService implements AsyncEventService {
List<QueueMessage> messagesToAck = handleMessages(messages);
if (messagesToAck == null || messagesToAck.size() == 0) {
+ logger.error("No messages came back from the queue operation",messages);
return messagesToAck;
}
+ if(messagesToAck.size()<messages.size()){
+ logger.error("Missing messages from queue post operation",messages,messagesToAck);
+ }
//ack each message, but only if we didn't error.
ack(messagesToAck);
+ //messagesToAck.stream().forEach(message -> ack(message));
return messagesToAck;
} catch (Exception e) {
logger.error("failed to ack messages to sqs", messages.get(0).getMessageId(), e);
[42/50] [abbrv] usergrid git commit: remove observable
Posted by sn...@apache.org.
remove observable
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/dbf37e48
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/dbf37e48
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/dbf37e48
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: dbf37e48caee2ca7fb5a2468f4800ad4bdf196ea
Parents: 9cbe283
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 13 11:14:21 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 13 11:14:21 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 114 +++++++++----------
1 file changed, 57 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/dbf37e48/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index f1a02ce..95126c6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import com.google.common.base.Optional;
import org.apache.usergrid.persistence.index.impl.IndexProducer;
@@ -248,7 +249,6 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
-
/**
* calls the event handlers and returns a result with information on whether it needs to be ack'd and whether it needs to be indexed
* @param messages
@@ -259,71 +259,71 @@ public class AmazonAsyncEventService implements AsyncEventService {
logger.debug("callEventHandlers with {} message", messages.size());
}
- Observable<IndexEventResult> masterObservable = Observable.from(messages).map(message -> {
- AsyncEvent event = null;
- try {
- event = (AsyncEvent) message.getBody();
- } catch (ClassCastException cce) {
- logger.error("Failed to deserialize message body", cce);
- }
-
- if (event == null) {
- logger.error("AsyncEvent type or event is null!");
- return new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(), System.currentTimeMillis());
- }
-
- final AsyncEvent thisEvent = event;
- if (logger.isDebugEnabled()) {
- logger.debug("Processing {} event", event);
- }
-
- try {
- Observable<IndexOperationMessage> indexoperationObservable;
- //merge each operation to a master observable;
- if (event instanceof EdgeDeleteEvent) {
- indexoperationObservable = handleEdgeDelete(message);
- } else if (event instanceof EdgeIndexEvent) {
- indexoperationObservable = handleEdgeIndex(message);
- } else if (event instanceof EntityDeleteEvent) {
- indexoperationObservable = handleEntityDelete(message);
- } else if (event instanceof EntityIndexEvent) {
- indexoperationObservable = handleEntityIndexUpdate(message);
- } else if (event instanceof InitializeApplicationIndexEvent) {
- //does not return observable
- handleInitializeApplicationIndex(event, message);
- indexoperationObservable = Observable.just(new IndexOperationMessage());
- } else {
- throw new Exception("Unknown EventType");//TODO: print json instead
+ Stream<IndexEventResult> indexEventResults = messages.stream()
+ .map(message -> {
+ AsyncEvent event = null;
+ try {
+ event = (AsyncEvent) message.getBody();
+ } catch (ClassCastException cce) {
+ logger.error("Failed to deserialize message body", cce);
}
- //collect all of the
- IndexOperationMessage indexOperationMessage =
- indexoperationObservable
- .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
- .toBlocking().lastOrDefault(null);
+ if (event == null) {
+ logger.error("AsyncEvent type or event is null!");
+ return new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(), System.currentTimeMillis());
+ }
- if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
- logger.info("Received empty index sequence message:({}), body:({}) ",
- message.getMessageId(),message.getStringBody());
+ final AsyncEvent thisEvent = event;
+ if (logger.isDebugEnabled()) {
+ logger.debug("Processing {} event", event);
}
- //return type that can be indexed and ack'd later
- return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
- } catch (Exception e) {
- logger.error("Failed to index message: " + message.getMessageId(), message.getStringBody() ,e);
- return new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime());
- }
- });
- //resolve the list and return it.
- final List<IndexEventResult> indexEventResults = masterObservable
- .collect(() -> new ArrayList<IndexEventResult>(), (list,indexEventResult) -> list.add(indexEventResult) )
- .toBlocking().lastOrDefault(null);
+ try {
+ //check for empty sets if this is true
+ boolean validateEmptySets = true;
+ Observable<IndexOperationMessage> indexoperationObservable;
+ //merge each operation to a master observable;
+ if (event instanceof EdgeDeleteEvent) {
+ indexoperationObservable = handleEdgeDelete(message);
+ } else if (event instanceof EdgeIndexEvent) {
+ indexoperationObservable = handleEdgeIndex(message);
+ } else if (event instanceof EntityDeleteEvent) {
+ indexoperationObservable = handleEntityDelete(message);
+ } else if (event instanceof EntityIndexEvent) {
+ indexoperationObservable = handleEntityIndexUpdate(message);
+ } else if (event instanceof InitializeApplicationIndexEvent) {
+ //does not return observable
+ handleInitializeApplicationIndex(event, message);
+ indexoperationObservable = Observable.just(new IndexOperationMessage());
+ validateEmptySets = false; //do not check this one for an empty set b/c it will be empty.
+ } else {
+ throw new Exception("Unknown EventType");//TODO: print json instead
+ }
+
+ //collect all of the
+ IndexOperationMessage indexOperationMessage =
+ indexoperationObservable
+ .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
+ .toBlocking().lastOrDefault(null);
+
+ if (validateEmptySets && (indexOperationMessage == null || indexOperationMessage.isEmpty())) {
+ logger.error("Received empty index sequence message:({}), body:({}) ",
+ message.getMessageId(), message.getStringBody());
+ throw new Exception("Received empty index sequence.");
+ }
+
+ //return type that can be indexed and ack'd later
+ return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
+ } catch (Exception e) {
+ logger.error("Failed to index message: " + message.getMessageId(), message.getStringBody(), e);
+ return new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime());
+ }
+ });
- return indexEventResults;
+ return indexEventResults.collect(Collectors.toList());
}
-
@Override
public void queueInitializeApplicationIndex( final ApplicationScope applicationScope) {
IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(
[19/50] [abbrv] usergrid git commit: remove integration tests
Posted by sn...@apache.org.
remove integration tests
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f005aceb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f005aceb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f005aceb
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: f005aceb5cf872bed8eba796b81219aca520d0dc
Parents: 9da0179
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 7 11:54:24 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 7 11:54:24 2015 -0600
----------------------------------------------------------------------
stack/rest_integration_tests/README.md | 19 ---
stack/rest_integration_tests/config/default.js | 39 -----
stack/rest_integration_tests/config/index.js | 25 ---
stack/rest_integration_tests/index.js | 20 ---
stack/rest_integration_tests/lib/connections.js | 137 ---------------
stack/rest_integration_tests/lib/entities.js | 167 -------------------
.../rest_integration_tests/lib/notifications.js | 68 --------
stack/rest_integration_tests/lib/random.js | 74 --------
stack/rest_integration_tests/lib/response.js | 30 ----
stack/rest_integration_tests/lib/token.js | 49 ------
stack/rest_integration_tests/lib/urls.js | 37 ----
stack/rest_integration_tests/lib/users.js | 82 ---------
stack/rest_integration_tests/package.json | 18 --
.../test/authentication/management.js | 38 -----
.../test/authentication/org.js | 37 ----
.../test/authentication/resetPassword.js | 78 ---------
.../test/authentication/user.js | 40 -----
.../test/connections/create.js | 49 ------
.../test/connections/delete.js | 77 ---------
.../test/connections/get.js | 82 ---------
.../test/entities/create.js | 73 --------
.../test/entities/deleteAll.js | 37 ----
.../rest_integration_tests/test/entities/get.js | 51 ------
.../test/entities/update.js | 43 -----
stack/rest_integration_tests/test/main.js | 72 --------
stack/rest_integration_tests/test/mocha.opts | 3 -
.../test/notifications/create.js | 36 ----
.../test/queries/comparison.js | 58 -------
.../test/queries/contains.js | 116 -------------
.../test/queries/equals.js | 54 ------
.../test/queries/location.js | 42 -----
.../test/queries/order.js | 82 ---------
stack/rest_integration_tests/test/setup.js | 103 ------------
stack/rest_integration_tests/test/teardown.js | 65 --------
.../rest_integration_tests/test/users/create.js | 45 -----
35 files changed, 2046 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/README.md
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/README.md b/stack/rest_integration_tests/README.md
deleted file mode 100644
index c27ec39..0000000
--- a/stack/rest_integration_tests/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-#Usergrid RESTful Integration Tests
-
-These tests will run against a deployed instance of Usergrid and validate that APIs respond as expected. Tests require [Node.js](https://nodejs.org), [Mocha](http://mochajs.org), and [Should.js](http://shouldjs.github.io).
-
-Get Started:
-
-1. Install [Node.js](https://nodejs.org/download) version 0.12.1 or newer
-2. Install Mocha: `$ [sudo] npm -g install mocha`
-3. `$ cd` to `/integration_tests` and run `$ npm install`.
-4. Using `config/default.js` as a template, create a copy `config/override.js` and modify it according to your environment.
-5. Once configured, run `$ mocha test` from `/integration_tests` to perform tests.
-
-Notes:
-
-- Connections do not currently support org/app credentials. For tests to pass, you will need to give `Guest` POST rights to `/**` in the Usergrid authorizations table.
-- In order for notifications tests to pass, you will need to create an Apple notifier named `apple-dev` using a valid development APNS certificate.
-- In order to skip tests, you can append `.skip` to the test method, e.g.: `describe.skip()` or `it.skip()`.
-- Depending on your environment, certain tests may take longer than expected. You can override timeouts by setting `this.timeout(timeInMilliseconds)` and `this.slow(timeInMilliseconds)` inside the `describe()` method before the tests execute.
-- For more information on adding or modifying tests, check out the [Mocha](http://mochajs.org), and [Should.js](http://shouldjs.github.io) documentation.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/config/default.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/config/default.js b/stack/rest_integration_tests/config/default.js
deleted file mode 100644
index f3b8c17..0000000
--- a/stack/rest_integration_tests/config/default.js
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-module.exports = {
- serverUrl: "http://localhost:8080/",
- orgName: "test-organization", //must
- appName: "test-app", //must pre create app
- numberOfUsers: 5,
- numberOfEntitiesConsistent: 100,
- consistentChecks:3,
- numberOfEntities: 20,
- org: {
- clientId: "",
- clientSecret: ""
- },
- usersCollection: "users",
- entitiesTestCollection: "cats",
- genericTestCollection1: "dogs",
- genericTestCollection2: "horses",
- consumableTestCollection: "food",
- location: { // London
- latitude: 51.51279,
- longitude: -0.09184
- },
- notifierName: "noop-dev"
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/config/index.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/config/index.js b/stack/rest_integration_tests/config/index.js
deleted file mode 100644
index 98ac92d..0000000
--- a/stack/rest_integration_tests/config/index.js
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var _ = require("underscore");
-var config = require("./default.js")
-var override = require("./override.js")
-
-
-if (Object.keys(override).length > 0) {
- config = _.extend(config, override);
-}
-module.exports = config;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/index.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/index.js b/stack/rest_integration_tests/index.js
deleted file mode 100644
index 30a47b0..0000000
--- a/stack/rest_integration_tests/index.js
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require('./config/config.js');
-
-console.log('config is '+ JSON.stringify(config));
-
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/connections.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/connections.js b/stack/rest_integration_tests/lib/connections.js
deleted file mode 100644
index 841281d..0000000
--- a/stack/rest_integration_tests/lib/connections.js
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-var urls = require("./urls");
-var random = require("./random");
-var responseLib = require("./response");
-var async = require('async');
-var request = require("request");
-var sleep = require('sleep');
-module.exports = {
- create: function(fromCollection, toCollection, relationship, cb) {
- async.parallel({
- from: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + fromCollection),
- json: true
- }, function(e, r, body) {
- cb(e, body.entities[0]);
- });
- },
- to: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + toCollection),
- json: true
- }, function(e, r, body) {
- cb(e, body.entities[0]);
- });
- }
- }, function(err, results) {
- var url = urls.appUrl() +
- fromCollection + "/" +
- results.from.uuid + "/" +
- relationship + "/" +
- results.to.uuid;
- url = urls.appendOrgCredentials(url)
- request.post({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- });
- });
- },
- get: function(fromCollection, toCollection, relationship, cb) {
- async.parallel({
- from: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + fromCollection + "?limit=1"),
- json: true
- }, function(e, r, body) {
- var o = {
- parent: body.entities[0]
- }
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + fromCollection + "/" + o.parent.uuid + "/" + relationship),
- json: true
- }, function(e, r, body) {
- o.related = body.entities;
- cb(e, o);
- });
- });
- },
- to: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + toCollection + "?limit=1"),
- json: true
- }, function(e, r, body) {
- var o = {
- parent: body.entities[0]
- }
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + toCollection + "/" + o.parent.uuid + "/connecting/" + relationship),
- json: true
- }, function(e, r, body) {
- o.related = body.entities;
- cb(e, o);
- });
- });
- }
- }, function(err, results) {
- cb(err, results);
- });
- },
- delete: function(fromCollection, toCollection, relationship, cb) {
- async.parallel({
- from: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + fromCollection),
- json: true
- }, function(e, r, body) {
- cb(e, body.entities[0]);
- });
- },
- to: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + toCollection),
- json: true
- }, function(e, r, body) {
- cb(e, body.entities[0]);
- });
- }
- }, function(err, results) {
- var url = urls.appUrl() +
- fromCollection + "/" +
- results.from.uuid + "/" +
- relationship + "/" +
- //toCollection + "/" +
- results.to.uuid;
- url = urls.appendOrgCredentials(url);
- sleep.sleep(1);
- request.del({
- url: url,
- json: true
- }, function(e, r, body) {
- sleep.sleep(1);
- module.exports.get(fromCollection, toCollection, relationship, function(err, results) {
- cb(err, results);
- });
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/entities.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/entities.js b/stack/rest_integration_tests/lib/entities.js
deleted file mode 100644
index fcdcfb9..0000000
--- a/stack/rest_integration_tests/lib/entities.js
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-var urls = require("./urls");
-var random = require("./random");
-var responseLib = require("./response");
-var async = require('async');
-var request = require("request");
-
-module.exports = {
- create: function(collection, numberOfEntities, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection);
- var requestArray = []
- geos = random.geo(config.location, 2000, numberOfEntities);
- // console.log(geos);
- for (var i = 0; i < numberOfEntities; i++) {
- requestArray.push({
- consistentProperty: "somethingConsistent",
- randomProperty: "somethingRandom - " + random.randomString(10),
- intProperty: random.randomNumber(5),
- optionsProperty: random.abc(),
- location: geos[i],
- title: "A Tale of Two Cities"
- });
- }
- request.post({
- url: url,
- json: true,
- body: requestArray
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- });
- },
- createEach: function(collection, numberOfEntities, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection);
- var requestArray = []
- geos = random.geo(config.location, 2000, numberOfEntities);
- // console.log(geos);
- for (var i = 0; i < numberOfEntities; i++) {
- requestArray.push({
- consistentProperty: "somethingConsistent",
- randomProperty: "somethingRandom - " + random.randomString(10),
- intProperty: random.randomNumber(5),
- optionsProperty: random.abc(),
- location: geos[i],
- title: "A Tale of Two Cities"
- });
- }
- var returnBody = [];
- async.each(requestArray, function(options, cb) {
- request.post({
- url: url,
- json: true,
- body: options
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- var entity = body && body.entities ? body.entities.pop() : null;
- entity && returnBody.push(entity);
- cb(error, error ? error : entity);
- });
- }, function(err,bodies) {
- cb(err,returnBody);
- });
-
- },
- deleteAll: function(collection, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection);
- deleteAllEntities(collection, function(e) {
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- })
- },
- update: function(collection, uuid, body, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection + "/" + uuid);
- request.put({
- url: url,
- body: body,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- },
- get: function(collection, numberOfEntities, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection + "?limit=" + numberOfEntities.toString());
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- },
- getByUuid: function(collection, uuid, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection + "/"+uuid);
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- },
- getWithQuery: function(collection, query, numberOfEntities, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection + "?ql=" + encodeURIComponent(query) + "&limit=" + numberOfEntities.toString());
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- }
-};
-
-function deleteAllEntities(collection, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection);
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- if (body.count === undefined) {
- cb("The 'count' property is not defined at " + url);
- } else if (body.count > 0) {
- var deletes = [];
- for (var i = 0; i < body.count; i++) {
- deletes.push({
- url: urls.appendOrgCredentials(urls.appUrl() + collection + "/" + body.entities[i].uuid),
- json: true
- });
- }
- async.each(deletes, function(options, cb) {
- request.del(options, function(e) {
- cb(e);
- });
- }, function(err) {
- setTimeout(function() {
- deleteAllEntities(collection, function(e) {
- cb(e);
- });
- }, 600); // Mandatory, since it seems to not retrieve entities if you make a request in < 600ms
- });
- } else {
- cb();
- }
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/notifications.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/notifications.js b/stack/rest_integration_tests/lib/notifications.js
deleted file mode 100644
index 7834e2c..0000000
--- a/stack/rest_integration_tests/lib/notifications.js
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-var urls = require("./urls");
-var random = require("./random");
-var responseLib = require("./response");
-var async = require('async');
-var request = require("request");
-var random = require("./random");
-
-
-module.exports = {
-
- create: function(message, cb) {
- var notifierName = config.notifierName + "_" + random.randomString(5);
-
- // Need to ensure at least one device exists in the devices collection
- request.post({
- url: urls.appendOrgCredentials(urls.appUrl() + "notifiers"),
- json: true,
- body: {
- name: notifierName,
- provider: "noop"
- }
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- if(error){
- return cb(error)
- }
- request.post({
- url: urls.appendOrgCredentials(urls.appUrl() + "devices"),
- json: true,
- body: {
- name: "testDevice"
- }
- }, function(e, r, body) {
- payload = {};
- payload[notifierName] = message;
- request.post({
- url: urls.appendOrgCredentials(urls.appUrl() + "devices;ql=/notifications"),
- json: true,
- body: {
- payloads: payload
- }
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- });
- });
- });
-
-
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/random.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/random.js b/stack/rest_integration_tests/lib/random.js
deleted file mode 100644
index d5dce65..0000000
--- a/stack/rest_integration_tests/lib/random.js
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-module.exports = {};
-module.exports.randomString = function randomString(length) {
- var text = "";
- var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
-
- for (var i = 0; i < length; i++) {
- text += possible.charAt(Math.floor(Math.random() * possible.length));
- }
- return text;
-}
-
-module.exports.abc = function abc() {
- letters = ["aaa bbb ccc", "ddd eee fff", "ggg hhh iii", "jjj kkk lll"];
- return letters[Math.floor(Math.random() * letters.length)];
-}
-
-module.exports.randomNumber = function randomNumber(length) {
- var text = "";
- var possible = "0123456789";
-
- for (var i = 0; i < length; i++) {
- text += possible.charAt(Math.floor(Math.random() * possible.length));
- }
- return parseInt(text);
-}
-
-module.exports.randomEntity = function randomEntity(entitiesArray) {
- return entitiesArray[Math.floor(Math.random()*entitiesArray.length)];
-}
-
-module.exports.geo = function geo(center, radius, count) {
- var points = [];
- for (var i = 0; i < count; i++) {
- points.push(randomGeo(center, radius));
- }
- return points;
-}
-
-function randomGeo(center, radius) {
- var y0 = center.latitude;
- var x0 = center.longitude;
- var rd = radius / 111300;
-
- var u = Math.random();
- var v = Math.random();
-
- var w = rd * Math.sqrt(u);
- var t = 2 * Math.PI * v;
- var x = w * Math.cos(t);
- var y = w * Math.sin(t);
-
- // var xp = x / Math.cos(y0);
-
- return {
- 'latitude': y + y0,
- 'longitude': x + x0
- };
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/response.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/response.js b/stack/rest_integration_tests/lib/response.js
deleted file mode 100644
index 7aab031..0000000
--- a/stack/rest_integration_tests/lib/response.js
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-module.exports = {};
-module.exports.getError = function(err, response) {
- return err || (response.statusCode >= 400 ? response.body : null)
-};
-
-module.exports.distanceInMeters = function(location1, location2) {
- var R = 6371000;
- var a = 0.5 - Math.cos((location2.latitude - location1.latitude) * Math.PI / 180) / 2 +
- Math.cos(location1.latitude * Math.PI / 180) * Math.cos(location2.latitude * Math.PI / 180) *
- (1 - Math.cos((location2.longitude - location1.longitude) * Math.PI / 180)) / 2;
-
- var distance = R * 2 * Math.asin(Math.sqrt(a));
- return distance;
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/token.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/token.js b/stack/rest_integration_tests/lib/token.js
deleted file mode 100644
index 96a3522..0000000
--- a/stack/rest_integration_tests/lib/token.js
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-var urls = require("./urls");
-var responseLib = require("./response");
-var request = require("request");
-
-module.exports= {
- getOrgToken: function (cb) {
- var managementUrl = urls.managementUrl();
-
- var options = {
- uri: managementUrl + "token",
- method: 'POST',
- json: {client_id: config.org.clientId, client_secret: config.org.clientSecret, grant_type: "client_credentials"}
- };
- request(options, function (err, response, body) {
- var error = responseLib.getError(err,response);
- cb(error, body);
- });
- },
- getManagementToken: function (username, password, cb) {
- var managementUrl = urls.managementUrl();
- var options = {
- uri: managementUrl + "token",
- method: 'POST',
- json: {username: username, password: password, grant_type: "password"}
- };
- request.post(options, function (err, response, body) {
- var error = responseLib.getError(err,response);
- cb(error,body);
- });
- }
-
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/urls.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/urls.js b/stack/rest_integration_tests/lib/urls.js
deleted file mode 100644
index 2edfd57..0000000
--- a/stack/rest_integration_tests/lib/urls.js
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-module.exports = {
- appUrl: function () {
- return config.serverUrl + config.orgName + "/" + config.appName + "/";
- },
- managementUrl: function () {
- return config.serverUrl + "management/";
- },
- appendAccessToken: function(url,tokenData){
- if(tokenData == null){
- return url;
- }
- var token = tokenData.access_token || tokenData;
- return url + (url.indexOf("?") >= 0 ? "&" : "?" ) + "access_token="+token;
- },
- appendOrgCredentials: function(url, clientId, clientSecret){
- clientId = clientId || config.org.clientId;
- clientSecret = clientSecret || config.org.clientSecret;
- return url + (url.indexOf("?") >= 0 ? "&" : "?" ) + "client_id="+clientId+"&client_secret="+clientSecret;
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/users.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/users.js b/stack/rest_integration_tests/lib/users.js
deleted file mode 100644
index 0a47324..0000000
--- a/stack/rest_integration_tests/lib/users.js
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var request = require("request");
-var token = require("./token");
-var urls = require("./urls");
-var responseLib = require("./response");
-module.exports = {};
-
-module.exports.add = function(user, cb) {
- request.post(urls.appendOrgCredentials(urls.appUrl() + "users"), {
- json: user
- }, function(err, response, body) {
- var error = responseLib.getError(err, response);
- cb(error, error ? null : body.entities.pop());
- });
-};
-
-module.exports.login = function(username, password, cb) {
- request.post({
- url: urls.appUrl() + "token",
- json: {
- username: username,
- password: password,
- grant_type: "password"
- }
- }, function(err, response, body) {
- var error = responseLib.getError(err, response);
- cb(error, body);
- });
-};
-
-module.exports.resetPassword = function(username, oldpassword, newpassword, cb) {
- request.post({
- uri: urls.appUrl() + "users/" + username + "/password",
- json: {
- oldpassword: oldpassword,
- newpassword: newpassword
- }
- }, function(e, r, body) {
- cb(e, r, body);
- });
-};
-
-module.exports.resetPasswordAsAdmin = function(username, newpassword, cb) {
- request.post({
- uri: urls.appendOrgCredentials(urls.appUrl() + "users/" + username + "/password"),
- json: {
- newpassword: newpassword
- }
- }, function(e, r, body) {
- cb(e, r, body);
- });
-};
-
-module.exports.addToRole = function(username, role, cb) {
- request.post(urls.appendOrgCredentials(urls.appUrl() + "roles/" + role + "/users/" + username), null, function(err, response, body) {
- var error = responseLib.getError(err, response);
- cb(error);
- });
-};
-
-module.exports.get = function(username, cb) {
- request.get(urls.appendOrgCredentials(urls.appUrl() + "users/" + username), function(err, response, body) {
- var json = JSON.parse(body);
- var error = response.statusCode === 404 ? null : responseLib.getError(err, response);
- cb(error, error ? null : response.statusCode === 404 ? null : json.entities.pop());
- })
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/package.json
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/package.json b/stack/rest_integration_tests/package.json
deleted file mode 100644
index bad9cbc..0000000
--- a/stack/rest_integration_tests/package.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "name": "Usergrid_Tests",
- "version": "0.0.1",
- "devDependencies": {
- "async": "^1.2.1",
- "colors": "~1.1.2",
- "i": "~0.3.3",
- "mocha": "~2.2.5",
- "request": "~2.58.0",
- "should": "~6.0.3",
- "sleep": "^3.0.0",
- "underscore": "^1.8.3",
- "uuid": "^2.0.1"
- },
- "engines": {
- "node": ">0.12.1"
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/authentication/management.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/authentication/management.js b/stack/rest_integration_tests/test/authentication/management.js
deleted file mode 100644
index bef2a96..0000000
--- a/stack/rest_integration_tests/test/authentication/management.js
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var users = require("../../lib/users")
-var config = require("../../config")
-var should = require("should")
-var setup = require("../setup")
-module.exports = {};
-
-module.exports.test = function() {
- describe('get a management token', function() {
- it('should return valid token', function(done) {
- var admin = setup.admins[0];
- users.login(admin.username, admin.password, function(err, body) {
- should(err).be.null;
- body.should.have.property('access_token').and.have.lengthOf(63);;
- body.should.have.property('expires_in');
- body.should.have.property('expires_in').which.is.a.Number;
- body.user.username.should.equal(admin.username);
- done();
- });
- });
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/authentication/org.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/authentication/org.js b/stack/rest_integration_tests/test/authentication/org.js
deleted file mode 100644
index 95d182a..0000000
--- a/stack/rest_integration_tests/test/authentication/org.js
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var token = require("../../lib/token")
-var config = require("../../config")
-var should = require("should")
-var setup = require("../setup")
-module.exports = {};
-
-module.exports.test = function() {
- describe('get an org token', function() {
- it('should return valid token', function(done) {
- token.getOrgToken(function(err, tokenData) {
- should(err).be.null;
- tokenData.should.have.property('access_token').and.have.lengthOf(63);;
- tokenData.should.have.property('expires_in');
- tokenData.should.have.property('expires_in').which.is.a.Number;
- tokenData.organization.name.should.equal(config.orgName)
- done();
- });
- });
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/authentication/resetPassword.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/authentication/resetPassword.js b/stack/rest_integration_tests/test/authentication/resetPassword.js
deleted file mode 100644
index 86d01a6..0000000
--- a/stack/rest_integration_tests/test/authentication/resetPassword.js
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var token = require("../../lib/token")
-var users = require("../../lib/users")
-var config = require("../../config")
-var should = require("should")
-var setup = require("../setup")
-var async = require("async");
-
-module.exports = {};
-
-module.exports.test = function() {
- describe('reset a user password', function() {
- it('should reset a user\'s password only when the correct old password is provided', function(done) {
- var user = setup.users[0];
- users.login(user.username, user.password, function(err, body) {
- should(err).be.null;
- body.should.have.property('access_token').and.have.lengthOf(63);
- async.parallel({
- bad: function(cb) {
- users.resetPassword(user.username, user.password + "_bad", user.password + "_badnew", function(e, r, body) {
- cb(e, {
- r: r,
- body: body
- });
- });
- },
- good: function(cb) {
- users.resetPassword(user.username, user.password, user.password + "_goodnew", function(e, r, body) {
- cb(e, {
- r: r,
- body: body
- });
- });
- }
- },
- function(err, results) {
- results.bad.r.statusCode.should.equal(400);
- results.bad.body.should.have.property('error').which.equal('auth_invalid_username_or_password');
- results.bad.body.should.have.property('exception').which.equal('org.apache.usergrid.management.exceptions.IncorrectPasswordException');
- results.bad.body.should.have.property('error_description').which.equal('Unable to authenticate due to username or password being incorrect');
-
- results.good.r.statusCode.should.equal(200);
- results.good.body.should.have.property('action').which.equal('set user password');
- results.good.body.should.have.property('duration');
-
- done();
- });
-
- });
- });
- it('should reset a user\'s password using org credentials', function(done) {
- var user = setup.users[0];
- users.resetPasswordAsAdmin(user.username, user.password, function(e, r, body) {
- r.statusCode.should.equal(200);
- body.should.have.property('action').which.equal('set user password');
- body.should.have.property('duration');
-
- done();
- });
- })
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/authentication/user.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/authentication/user.js b/stack/rest_integration_tests/test/authentication/user.js
deleted file mode 100644
index 8faddd3..0000000
--- a/stack/rest_integration_tests/test/authentication/user.js
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var token = require("../../lib/token")
-var users = require("../../lib/users")
-var config = require("../../config")
-var should = require("should")
-var setup = require("../setup")
-
-module.exports = {};
-
-module.exports.test = function() {
- describe('get a user token', function() {
- it('should return valid token', function(done) {
- var user = setup.users[0];
- users.login(user.username, user.password, function(err, body) {
- should(err).be.null;
- body.should.have.property('access_token').and.have.lengthOf(63);
- body.should.have.property('expires_in');
- body.should.have.property('expires_in').which.is.a.Number;
- body.user.username.should.equal(user.username);
- done();
- });
- });
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/connections/create.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/connections/create.js b/stack/rest_integration_tests/test/connections/create.js
deleted file mode 100644
index ca9ce84..0000000
--- a/stack/rest_integration_tests/test/connections/create.js
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var connections = require("../../lib/connections");
-var should = require("should");
-var config = require('../../config');
-var inflect = require('i')();
-
-module.exports = {
- test: function() {
- describe("create connection", function() {
- it("should connect " + config.genericTestCollection1 + "[0] to " + config.consumableTestCollection + "[0] via the relationship 'consumed'",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.create(config.genericTestCollection1, config.consumableTestCollection, "consumed", function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.entities[0].type.should.equal(config.consumableTestCollection);
- done();
- })
- });
- it("should connect " + config.genericTestCollection1 + "[0] to " + config.genericTestCollection2 + "[0] via the relationship 'likes'",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.create(config.genericTestCollection1, config.genericTestCollection2, "likes", function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.entities[0].type.should.equal(inflect.singularize(config.genericTestCollection2));
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/connections/delete.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/connections/delete.js b/stack/rest_integration_tests/test/connections/delete.js
deleted file mode 100644
index 9014a02..0000000
--- a/stack/rest_integration_tests/test/connections/delete.js
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var connections = require("../../lib/connections");
-var should = require("should");
-var config = require('../../config');
-var util = require('util');
-var inflect = require('i')();
-
-module.exports = {
- test: function() {
- describe("delete connections", function() {
- var rel1 = "consumed";
- it("should delete the '" + rel1 + "' connection between " + config.genericTestCollection1 + "[0] and " + config.consumableTestCollection + "[0]",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.delete(config.genericTestCollection1, config.consumableTestCollection, rel1, function(err, r) {
- should(err).be.null;
- if (r.from.parent.metadata.hasOwnProperty("connections")) {
- r.from.parent.metadata.connections.should.not.have.property(rel1);
- } else {
- r.from.parent.metadata.should.not.have.property("connections");
- }
- r.from.parent.metadata.should.not.have.property("connecting");
- r.from.related.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- if (r.to.parent.metadata.hasOwnProperty("connecting")) {
- r.to.parent.metadata.connecting.should.not.have.property(rel1);
- } else {
- r.to.parent.metadata.should.not.have.property("connecting");
- }
- r.to.related.should.be.an.instanceOf(Array).and.have.lengthOf(0);
-
- done();
- })
- });
- var rel2 = "likes";
- it("should delete the '" + rel2 + "' connection between " + config.genericTestCollection1 + "[0] and " + config.genericTestCollection2 + "[0]",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.delete(config.genericTestCollection1, config.genericTestCollection2, rel2, function(err, r) {
- should(err).be.null;
- if (r.from.parent.metadata.hasOwnProperty("connections")) {
- r.from.parent.metadata.connections.should.not.have.property(rel2);
- } else {
- r.from.parent.metadata.should.not.have.property("connections");
- }
- r.from.parent.metadata.should.not.have.property("connecting");
- r.from.related.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- if (r.to.parent.metadata.hasOwnProperty("connecting")) {
- r.to.parent.metadata.connecting.should.not.have.property(rel2);
- } else {
- r.to.parent.metadata.should.not.have.property("connecting");
- }
- r.from.parent.metadata.should.not.have.property("connections");
- r.to.related.should.be.an.instanceOf(Array).and.have.lengthOf(0);
-
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/connections/get.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/connections/get.js b/stack/rest_integration_tests/test/connections/get.js
deleted file mode 100644
index cea7bbe..0000000
--- a/stack/rest_integration_tests/test/connections/get.js
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var connections = require("../../lib/connections");
-var should = require("should");
-var config = require('../../config');
-var util = require('util');
-var inflect = require('i')();
-
-module.exports = {
- test: function() {
- describe("get connections", function() {
- var rel1 = "consumed";
- it("should see " + config.genericTestCollection1 + "[0] connected to " + config.consumableTestCollection + "[0] by the relationship '" + rel1 + "'",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.get(config.genericTestCollection1, config.consumableTestCollection, rel1, function(err, r) {
- should(err).be.null;
-
- r.from.parent.metadata.connections.should.have.property(rel1)
- r.from.parent.metadata.connections[rel1].should.equal(
- util.format("/%s/%s/%s", inflect.pluralize(config.genericTestCollection1), r.from.parent.uuid, rel1)
- );
- r.from.parent.type.should.equal(inflect.singularize(config.genericTestCollection1));
- r.from.related[0].uuid.should.equal(r.to.parent.uuid);
- r.from.related[0].type.should.equal(inflect.singularize(config.consumableTestCollection));
-
- r.to.parent.metadata.connecting.should.have.property(rel1)
- r.to.parent.metadata.connecting[rel1].should.equal(
- util.format("/%s/%s/connecting/%s", inflect.pluralize(config.consumableTestCollection), r.to.parent.uuid, rel1)
- );
- r.to.parent.type.should.equal(inflect.singularize(config.consumableTestCollection));
- r.to.related[0].uuid.should.equal(r.from.parent.uuid);
- r.to.related[0].type.should.equal(inflect.singularize(config.genericTestCollection1));
-
- done();
- })
- });
- var rel2 = "likes";
- it("should see " + config.genericTestCollection1 + "[0] connected to " + config.genericTestCollection2 + "[0] by the relationship '" + rel2 + "'",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.get(config.genericTestCollection1, config.genericTestCollection2, rel2, function(err, r) {
- should(err).be.null;
-
- r.from.parent.metadata.connections.should.have.property(rel2)
- r.from.parent.metadata.connections[rel2].should.equal(
- util.format("/%s/%s/%s", inflect.pluralize(config.genericTestCollection1), r.from.parent.uuid, rel2)
- );
- r.from.parent.type.should.equal(inflect.singularize(config.genericTestCollection1));
- r.from.related[0].uuid.should.equal(r.to.parent.uuid);
- r.from.related[0].type.should.equal(inflect.singularize(config.genericTestCollection2));
-
- r.to.parent.metadata.connecting.should.have.property(rel2)
- r.to.parent.metadata.connecting[rel2].should.equal(
- util.format("/%s/%s/connecting/%s", inflect.pluralize(config.genericTestCollection2), r.to.parent.uuid, rel2)
- );
- r.to.parent.type.should.equal(inflect.singularize(config.genericTestCollection2));
- r.to.related[0].uuid.should.equal(r.from.parent.uuid);
- r.to.related[0].type.should.equal(inflect.singularize(config.genericTestCollection1));
-
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/entities/create.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/entities/create.js b/stack/rest_integration_tests/test/entities/create.js
deleted file mode 100644
index faccee8..0000000
--- a/stack/rest_integration_tests/test/entities/create.js
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var config = require('../../config');
-var async = require('async');
-module.exports = {
- test: function() {
- var uuid = require("uuid");
- var collectionName = "resttest_"+ uuid.v1().toString().replace("-", "");
-
- describe("create entities", function() {
- var numberOfRecords = config.numberOfEntities;
-
- it("should create " + numberOfRecords.toString() + " entities in the " + config.entitiesTestCollection + " collection", function(done) {
- this.slow(numberOfRecords * 500);
- entities.create(config.entitiesTestCollection, numberOfRecords, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(numberOfRecords);
- body.entities.forEach(function(entity) {
- entity.should.have.property("uuid").and.match(/(\w{8}(-\w{4}){3}-\w{12}?)/);
- });
- done();
- })
- });
- var numberOfRecordsConsistent = config.numberOfEntitiesConsistent;
- it("should create " + numberOfRecordsConsistent.toString() + " entities in the " + collectionName + " collection and check for consistency", function(done) {
- this.timeout(60000*100);
- bodyMap = {};
-
- //this.slow(numberOfRecordsConsistent * 500);
- async.times(config.consistentChecks, function(n, next) {
- entities.createEach(collectionName+n, numberOfRecordsConsistent, function (err, bodies) {
- should(err).be.null;
- bodies.should.be.an.instanceOf(Array).and.have.lengthOf(numberOfRecordsConsistent);
- bodies.forEach(function (body) {
- bodyMap[body.uuid] = body;
- });
- entities.get(collectionName+n, numberOfRecordsConsistent, function (err, entityArray) {
- should(err).be.null;
- next(err,entityArray);
- });
- });
- },
- function(err,list){
- list.forEach(function (listOfEntities) {
- listOfEntities.entities.forEach(function(entity){
- delete(bodyMap[entity.uuid]);
- });
- });
- Object.keys(bodyMap).length && console.log(JSON.stringify(bodyMap));
- should(Object.keys(bodyMap)).have.lengthOf(0);
- done();
- }
- );
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/entities/deleteAll.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/entities/deleteAll.js b/stack/rest_integration_tests/test/entities/deleteAll.js
deleted file mode 100644
index daa20ea..0000000
--- a/stack/rest_integration_tests/test/entities/deleteAll.js
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var config = require('../../config');
-
-module.exports = {
- test: function(collectionName) {
- collectionName = collectionName ? collectionName : config.entitiesTestCollection;
- describe("delete entities", function() {
- it("should delete all entities from the " + collectionName + " collection", function(done) {
- this.timeout(60000);
- this.slow(30000);
- entities.deleteAll(collectionName, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/entities/get.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/entities/get.js b/stack/rest_integration_tests/test/entities/get.js
deleted file mode 100644
index 665b1e6..0000000
--- a/stack/rest_integration_tests/test/entities/get.js
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require("async");
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("get entities", function() {
- it("should get 1 entity", function(done) {
- entities.get(config.entitiesTestCollection, 1, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.count.should.equal(1);
- done();
- })
- });
- it("should get 4 entities", function(done) {
- entities.get(config.entitiesTestCollection, 4, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(4);
- body.count.should.equal(4);
- done();
- })
- });
- it("should get 18 entities", function(done) {
- entities.get(config.entitiesTestCollection, 18, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(18);
- body.count.should.equal(18);
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/entities/update.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/entities/update.js b/stack/rest_integration_tests/test/entities/update.js
deleted file mode 100644
index 04f4a97..0000000
--- a/stack/rest_integration_tests/test/entities/update.js
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var random = require("../../lib/random");
-var should = require("should");
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("update entity", function() {
- it("should get a random entity and set 'newProperty' to 'BANJO'", function(done) {
- this.timeout(10000);
- this.slow(5000);
- entities.get(config.entitiesTestCollection, random.randomNumber(10), function(err, body) {
- var payload = {
- newProperty: "BANJO"
- }
- should(body.entities[0].newProperty).not.exist;
- entities.update(config.entitiesTestCollection, body.entities[body.entities.length - 1].uuid, payload, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.entities[0].newProperty.should.equal("BANJO");
- done();
- });
- });
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/main.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/main.js b/stack/rest_integration_tests/test/main.js
deleted file mode 100644
index 24b6cfe..0000000
--- a/stack/rest_integration_tests/test/main.js
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var should = require("should");
-var config = require("../config/");
-var setup = require("./setup");
-var teardown = require("./teardown");
-var async = require('async');
-var request = require('request');
-var colors = require('colors');
-
-var entities = require('../lib/entities');
-
-describe("baas 2.0 tests", function() {
- before(function(done) {
- setup.do(function() {
- done();
- })
- });
- describe("authentication", function() {
- require("./authentication/user.js").test();
- require("./authentication/resetPassword.js").test();
- require("./authentication/management.js").test();
- require("./authentication/org.js").test();
- });
- describe("users", function() {
- require("./users/create.js").test();
- });
- describe("entities", function() {
- require("./entities/create.js").test();
- require("./entities/get.js").test();
- require("./entities/update.js").test();
- require("./entities/deleteAll.js").test();
- });
- describe("connections", function() {
- require('./connections/create.js').test();
- require('./connections/get.js').test();
- require('./connections/delete.js').test();
- });
- describe("queries", function() {
- require('./queries/equals.js').test();
- require('./queries/contains.js').test();
- require('./queries/order.js').test();
- require('./queries/comparison.js').test();
- require('./queries/location.js').test();
- });
- describe("notifications", function() {
- // Requires an apple notifier to be created in BaaS portal prior to running this test.
- // See: http://apigee.com/docs/app-services/content/creating-notifiers
- require('./notifications/create.js').test();
- });
-
- after(function(done) {
- this.timeout(40000);
- teardown.do(function() {
- done();
- });
- });
-});
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/mocha.opts
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/mocha.opts b/stack/rest_integration_tests/test/mocha.opts
deleted file mode 100644
index 9e2480f..0000000
--- a/stack/rest_integration_tests/test/mocha.opts
+++ /dev/null
@@ -1,3 +0,0 @@
---ui bdd
---recursive
---timeout 5000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/notifications/create.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/notifications/create.js b/stack/rest_integration_tests/test/notifications/create.js
deleted file mode 100644
index b3a2bb7..0000000
--- a/stack/rest_integration_tests/test/notifications/create.js
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var should = require("should");
-var uuid = require("uuid");
-var notifications = require("../../lib/notifications");
-
-module.exports = {
- test: function() {
- // Requires an apple notifier to be created in BaaS portal prior to running this test.
- // See: http://apigee.com/docs/app-services/content/creating-notifiers
- describe("create a notification", function() {
- it("should successfully create a notification", function(done) {
- notifications.create("Hello World!", function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.entities[0].state.should.equal('FINISHED');
- done();
- });
- });
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/comparison.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/comparison.js b/stack/rest_integration_tests/test/queries/comparison.js
deleted file mode 100644
index 5fedbdb..0000000
--- a/stack/rest_integration_tests/test/queries/comparison.js
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("filter " + config.genericTestCollection2 + " with '>' and '<' queries", function() {
- var query = "where intProperty > 30000";
- numberOfEntities = Math.min(config.numberOfEntities, 10);
- it('should return a subset of results ' + query, function(done) {
- this.timeout(10000);
- entities.getWithQuery(config.genericTestCollection2, query, numberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.intProperty.should.be.greaterThan(30000);
- });
- }
- done();
- });
- });
- var query = "where intProperty > 30000 && intProperty < 40000";
- it('should return a subset of results ' + query, function(done) {
- this.timeout(10000);
- entities.getWithQuery(config.genericTestCollection2, query, numberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.intProperty.should.be.greaterThan(30000).and.lessThan(40000);
- });
- }
- done();
- });
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/contains.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/contains.js b/stack/rest_integration_tests/test/queries/contains.js
deleted file mode 100644
index 01c8ef1..0000000
--- a/stack/rest_integration_tests/test/queries/contains.js
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("filter " + config.genericTestCollection1 + " with 'contains' queries", function(done) {
- var query1 = "where consistentProperty contains 'somethingConsistent'";
- var maxNumberOfEntities = Math.max(config.numberOfEntities, 100);
- it('should return ' + config.numberOfEntities + ' results ' + query1, function(done) {
- this.timeout(30000);
- entities.getWithQuery(config.genericTestCollection1, query1, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.equal(config.numberOfEntities);
- body.entities.forEach(function(entity) {
- entity.consistentProperty.should.equal('somethingConsistent');
- });
- done();
- });
- });
- var query2 = "where consistentProperty contains '*ethi*'";
- // skipping this test for now since it doesn't work in 1.0
- it.skip('should return ' + config.numberOfEntities + ' results ' + query2, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query2, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.equal(config.numberOfEntities);
- body.entities.forEach(function(entity) {
- entity.consistentProperty.should.equal('somethingConsistent');
- });
- done();
- });
- });
- var query3 = "where optionsProperty contains 'aaa*'";
- // this should be updated when running tests against 2.0 - *aaa* instead of aaa*
- it('should return a subset of results ' + query3, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query3, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(config.numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.optionsProperty.should.match(/(\b|^)aaa(\b|$)/);
- });
- }
- done();
- });
- });
- var query4 = "where title contains 'tale'";
- it('should return a subset of results ' + query4, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query4, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(config.numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.title.should.match(/tale/i);
- });
- }
- done();
- });
- });
- var query5 = "where title contains 'ta*'";
- it('should return a subset of results ' + query5, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query5, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(config.numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.title.should.match(/ta.*/i);
- });
- }
- done();
- });
- });
- var query6 = "where consistentProperty contains 'some*'";
- it('should return a subset of results ' + query6, function() {
- entities.getWithQuery('horses', query6, 10, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.be.greaterThan(0).and.lessThan(11);
- });
- });
- var query7 = "where consistentProperty contains 'ssccxxome*'";
- it('should not return a subset of results ' + query7, function() {
- var query = "where firstProperty contains 'ssccxxome*'";
- entities.getWithQuery('horses', query7, 10, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.be.equal(0);
-
- });
- });
- });
- }
-}
[18/50] [abbrv] usergrid git commit: remove integration tests
Posted by sn...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/equals.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/equals.js b/stack/rest_integration_tests/test/queries/equals.js
deleted file mode 100644
index 904646b..0000000
--- a/stack/rest_integration_tests/test/queries/equals.js
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("filter " + config.genericTestCollection1 + " with '=' and '!=' queries", function(done) {
- var query1 = "where consistentProperty = 'somethingConsistent'";
- maxNumberOfEntities = Math.max(config.numberOfEntities, 100);
- it('should return ' + config.numberOfEntities + ' results ' + query1, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query1, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.equal(config.numberOfEntities);
- body.entities.forEach(function(entity) {
- entity.consistentProperty.should.equal('somethingConsistent');
- });
- done();
- });
- });
-
- var query2 = "where title = 'A Tale of Two Cities'";
- maxNumberOfEntities = Math.max(config.numberOfEntities, 100);
- it('should return ' + config.numberOfEntities + ' results ' + query2, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query2, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.equal(config.numberOfEntities);
- body.entities.forEach(function(entity) {
- entity.title.should.equal('A Tale of Two Cities');
- });
- done();
- });
- });
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/location.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/location.js b/stack/rest_integration_tests/test/queries/location.js
deleted file mode 100644
index 6fbc96a..0000000
--- a/stack/rest_integration_tests/test/queries/location.js
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-var response = require('../../lib/response');
-
-module.exports = {
- test: function() {
- describe("filter " + config.genericTestCollection2 + " with location queries", function(done) {
- var locationString = config.location.latitude + ", " + config.location.longitude;
- var query = "location within 1000 of " + locationString;
- maxNumberOfEntities = Math.max(config.numberOfEntities, 100);
- it("should return all results with a location within 1000m of " + locationString, function(done) {
- entities.getWithQuery(config.genericTestCollection2, query, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.forEach(function(entity) {
- var distance = response.distanceInMeters(config.location, entity.location);
- distance.should.be.lessThan(1000);
- });
- done();
- });
- });
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/order.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/order.js b/stack/rest_integration_tests/test/queries/order.js
deleted file mode 100644
index 8cbb78f..0000000
--- a/stack/rest_integration_tests/test/queries/order.js
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("sort " + config.genericTestCollection1 + " with 'order by' queries", function(done) {
- var query1 = "order by created desc";
- it('should return a subset of results ' + query1.replace('order', 'ordered'), function(done) {
- entities.getWithQuery(config.genericTestCollection1, query1, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- var comparisonArray = body.entities;
- comparisonArray.sort(function(a, b) {
- return a.created + b.created;
- });
- body.entities.should.equal(comparisonArray);
- done();
- });
- });
- var query2 = "order by created asc";
- it('should return a subset of results ' + query2.replace('order', 'ordered'), function(done) {
- entities.getWithQuery(config.genericTestCollection1, query2, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- var comparisonArray = body.entities;
- comparisonArray.sort(function(a, b) {
- return a.created - b.created;
- });
- body.entities.should.equal(comparisonArray);
- done();
- });
- });
- var query3 = "order by optionsProperty desc";
- it('should return a subset of results ' + query3.replace('order', 'ordered'), function(done) {
- entities.getWithQuery(config.genericTestCollection1, query3, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- var comparisonArray = body.entities;
- comparisonArray.sort(function(a, b) {
- if (a.optionsProperty > b.optionsProperty) return -1;
- if (a.optionsProperty < b.optionsProperty) return 1;
- return 0;
- });
- body.entities.should.equal(comparisonArray);
- done();
- });
- });
- var query4 = "order by optionsProperty asc";
- it('should return a subset of results ' + query4.replace('order', 'ordered'), function(done) {
- entities.getWithQuery(config.genericTestCollection1, query4, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- var comparisonArray = body.entities;
- comparisonArray.sort(function(a, b) {
- if (a.optionsProperty < b.optionsProperty) return -1;
- if (a.optionsProperty > b.optionsProperty) return 1;
- return 0;
- });
- done();
- });
- });
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/setup.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/setup.js b/stack/rest_integration_tests/test/setup.js
deleted file mode 100644
index 4fbc664..0000000
--- a/stack/rest_integration_tests/test/setup.js
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-// here we're going to do teardown of BaaS environment - deletion of users, entities, etc.
-var users = require("../lib/users");
-var entities = require("../lib/entities");
-var config = require("../config");
-var async = require("async");
-var uuid = require("uuid");
-var should = require("should");
-
-module.exports = {
- users: [],
- admins: [],
- do: function(cb) {
- console.log(" setup");
- async.parallel([
- function(cb) {
- // create admin user
- var id = uuid.v1().toString().replace("-", "");
- var admin = {
- username: id + "user",
- password: "pwd" + id,
- name: id + " name",
- email: id + "@uge2e.com"
- }
- users.add(admin, function(err, user) {
- users.addToRole(user.username, "admin", function(err) {
- module.exports.admins.push(admin);
- cb(err, err ? null : admin);
- });
- });
- },
- function(cb) {
- // create app users
- var size = config.numberOfUsers;
- var userArr = [];
- module.exports.users = userArr;
- for (var i = 0; i < size; i++) {
- var id = uuid.v1().toString().replace("-", "");
- userArr.push({
- username: id + "user",
- password: "pwd" + id,
- name: id + " name",
- email: id + "@uge2e.com"
- });
- }
- async.each(
- userArr,
- function(user, cb) {
- users.add(user, function(err, user) {
- cb(err, user);
- });
- },
- function(err, localUsers) {
- cb(err);
- }
- )
- },
- function(cb) {
- // create entities
- async.parallel([
- function(cb) {
- entities.create(config.genericTestCollection1, config.numberOfEntities, function(err, body) {
- cb(err);
- });
- },
- function(cb) {
- entities.create(config.genericTestCollection2, config.numberOfEntities, function(err, body) {
- cb(err);
- });
- },
- function(cb) {
- entities.create(config.consumableTestCollection, config.numberOfEntities, function(err, body) {
- cb(err);
- });
- }
- ],
- function(err, data) {
- cb(err);
- });
- }
- ],
- function(err, data) {
- should(err).be.null;
- console.log(" ✓".green + " done".grey);
- if (cb && typeof(cb) === 'function') cb(err);
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/teardown.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/teardown.js b/stack/rest_integration_tests/test/teardown.js
deleted file mode 100644
index 7afdc97..0000000
--- a/stack/rest_integration_tests/test/teardown.js
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../lib/entities");
-var should = require("should");
-var async = require("async");
-var config = require("../config");
-var inflect = require("i")();
-
-module.exports = {
- do: function(cb) {
- console.log(" teardown");
- async.parallel([
- function(cb) {
- entities.deleteAll(config.usersCollection, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- cb(err);
- })
- },
- function(cb) {
- entities.deleteAll(config.genericTestCollection1, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- cb(err);
- })
- },
- function(cb) {
- entities.deleteAll(config.genericTestCollection2, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- cb(err);
- })
- },
- function(cb) {
- entities.deleteAll(inflect.pluralize(config.consumableTestCollection), function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- cb(err);
- })
- }
- ],
- function(err, data) {
- console.log(" ✓".green + " done".grey);
- if (cb && typeof(cb) === 'function') cb(err);
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/users/create.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/users/create.js b/stack/rest_integration_tests/test/users/create.js
deleted file mode 100644
index c62ccc4..0000000
--- a/stack/rest_integration_tests/test/users/create.js
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var should = require("should");
-var uuid = require("uuid");
-var users = require("../../lib/users");
-
-module.exports = {
- test: function() {
- describe("create a user", function() {
- var username = "testuser" + uuid.v1();
- var password = "password";
- it("should successfully return a user object", function(done) {
- users.add({
- username: username,
- password: password,
- name: username + " name",
- email: username + "@uge2e.com"
- }, function(err, userBody) {
- should(err).be.null;
- userBody.should.not.be.null;
- users.login(username, password, function(err, body) {
- should(err).be.null;
- body.should.not.be.null;
- body.should.have.property("access_token");
- done();
- })
- });
- });
- });
- }
-}
[41/50] [abbrv] usergrid git commit: add better logging for empty
sequences
Posted by sn...@apache.org.
add better logging for empty sequences
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9cbe283d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9cbe283d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9cbe283d
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 9cbe283d0af45b877fb93dfae81cdb343cec26c5
Parents: a2a07aa
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 12 16:27:13 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 12 16:27:13 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 13 ++++++-----
.../asyncevents/EventBuilderImpl.java | 3 ++-
.../core/rx/ExceptionBehaviorTest.java | 24 ++++++++++++++++++++
.../persistence/queue/QueueMessage.java | 10 ++++++++
.../queue/impl/SNSQueueManagerImpl.java | 2 ++
.../queue/impl/SQSQueueManagerImpl.java | 1 +
6 files changed, 46 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 45d83cd..f1a02ce 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -299,17 +299,18 @@ public class AmazonAsyncEventService implements AsyncEventService {
//collect all of the
IndexOperationMessage indexOperationMessage =
indexoperationObservable
- .collect(() -> new IndexOperationMessage(), (collector, single ) -> collector.ingest(single))
+ .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
.toBlocking().lastOrDefault(null);
- if(indexOperationMessage == null){
- throw new IllegalArgumentException("Received null index operation.");
+ if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
+ logger.info("Received empty index sequence message:({}), body:({}) ",
+ message.getMessageId(),message.getStringBody());
}
//return type that can be indexed and ack'd later
return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
} catch (Exception e) {
- logger.error("Failed to index message: " + message.getMessageId(), e, message);
+ logger.error("Failed to index message: " + message.getMessageId(), message.getStringBody() ,e);
return new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime());
}
});
@@ -559,7 +560,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
List<IndexEventResult> indexEventResults = callEventHandlers(messages);
List<QueueMessage> messagesToAck = submitToIndex(indexEventResults);
if (messagesToAck == null || messagesToAck.size() == 0) {
- logger.error("No messages came back from the queue operation",messages);
+ logger.error("No messages came back from the queue operation should have seen "+messages.size(),messages);
return messagesToAck;
}
if(messagesToAck.size()<messages.size()){
@@ -569,7 +570,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
ack(messagesToAck);
return messagesToAck;
} catch (Exception e) {
- logger.error("failed to ack messages to sqs", messages, e);
+ logger.error("failed to ack messages to sqs", e);
return null;
//do not rethrow so we can process all of them
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 4bf5695..cc0356b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -181,8 +181,9 @@ public class EventBuilderImpl implements EventBuilder {
return true;
}
+ //entityIndexOperation.getUpdatedSince will always be 0 except for reindexing the application
//only re-index if it has been updated and been updated after our timestamp
- return modified.getValue() >= entityIndexOperation.getUpdatedSince();
+ return modified.getValue() >= entityIndexOperation.getUpdatedSince();
} )
//perform indexing on the task scheduler and start it
.flatMap( entity -> indexService.indexEntity( applicationScope, entity ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
index 8e4f4c4..cb39ca1 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
@@ -18,12 +18,16 @@
package org.apache.usergrid.persistence.core.rx;
+import org.junit.Assert;
import org.junit.Test;
import rx.Observable;
import rx.Observer;
import rx.schedulers.Schedulers;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* Tests RX exception behavior
@@ -47,6 +51,26 @@ public class ExceptionBehaviorTest {
} ).toBlocking().last();
}
+ @Test()
+ public void testSequence(){
+ ArrayList listReturn = Observable.range(0, 1).flatMap(i -> Observable.empty())
+ .collect(()->new ArrayList(),(list,i) ->{
+ list.add(i);
+ }).toBlocking().lastOrDefault(null);
+
+ Assert.assertEquals(listReturn,new ArrayList<Integer>());
+ }
+
+ @Test()
+ public void testSequence2(){
+ ArrayList listReturn = Observable.range(0, 2).buffer(2).flatMap(i -> Observable.empty())
+ .collect(()->new ArrayList(),(list,i) ->{
+ list.add(i);
+ }).toBlocking().lastOrDefault(null);
+
+ Assert.assertEquals(listReturn,new ArrayList<Integer>());
+ }
+
//
// /**
// * This shows that no re-throw happens on subscribe. This is as designed, but not as expected
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java
index 0874e9c..55f79f4 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java
@@ -22,6 +22,7 @@ public class QueueMessage {
private final String messageId;
private final String handle;
private final String type;
+ private String stringBody;
public QueueMessage(String messageId, String handle, Object body,String type) {
@@ -29,6 +30,7 @@ public class QueueMessage {
this.messageId = messageId;
this.handle = handle;
this.type = type;
+ this.stringBody = "";
}
public String getHandle() {
@@ -47,4 +49,12 @@ public class QueueMessage {
public String getType() {
return type;
}
+
+ public void setStringBody(String stringBody) {
+ this.stringBody = stringBody;
+ }
+
+ public String getStringBody() {
+ return stringBody;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index bc63f53..a2b5d72 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -364,6 +364,7 @@ public class SNSQueueManagerImpl implements QueueManager {
for (Message message : messages) {
Object body;
+ final String originalBody = message.getBody();
try {
final JsonNode bodyNode = mapper.readTree(message.getBody());
@@ -375,6 +376,7 @@ public class SNSQueueManagerImpl implements QueueManager {
}
QueueMessage queueMessage = new QueueMessage(message.getMessageId(), message.getReceiptHandle(), body, message.getAttributes().get("type"));
+ queueMessage.setStringBody(originalBody);
queueMessages.add(queueMessage);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index daa1cb5..fa9a7ac 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -196,6 +196,7 @@ public class SQSQueueManagerImpl implements QueueManager {
}
QueueMessage queueMessage = new QueueMessage(message.getMessageId(), message.getReceiptHandle(), body, message.getAttributes().get("type"));
+ queueMessage.setStringBody(message.getBody());
queueMessages.add(queueMessage);
}
[12/50] [abbrv] usergrid git commit: commit stash
Posted by sn...@apache.org.
commit stash
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7c5a864d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7c5a864d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7c5a864d
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 7c5a864dd1ce22ada5456ced1a6c10f9a5533b1f
Parents: 7dceb56
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 6 14:34:33 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 6 14:34:33 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 71 ++++++++++++++------
.../index/impl/EsIndexProducerImpl.java | 6 +-
2 files changed, 53 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7c5a864d/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 14d37b5..1bc70cd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.base.Optional;
@@ -226,6 +227,28 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
+ /**
+ * Ack message in SQS
+ */
+ public void ack(final List<QueueMessage> messages) {
+
+ final Timer.Context timer = this.ackTimer.time();
+
+ try{
+ queue.commitMessages(messages);
+
+ //decrement our in-flight counter
+ inFlight.decrementAndGet();
+
+ }catch(Exception e){
+ throw new RuntimeException("Unable to ack messages", e);
+ }finally {
+ timer.stop();
+ }
+
+
+ }
+
private Observable<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
if (logger.isDebugEnabled()) {
@@ -243,7 +266,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
if (event == null) {
logger.error("AsyncEvent type or event is null!");
- return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), true));
+ return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent()));
}
try {
//merge each operation to a master observable;
@@ -258,23 +281,24 @@ public class AmazonAsyncEventService implements AsyncEventService {
} else if (event instanceof InitializeApplicationIndexEvent) {
//does not return observable
handleInitializeApplicationIndex(message);
- return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), false));
+ return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent()));
} else {
- logger.error("Unknown EventType: {}", event);
- return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), true));
+ logger.error("Unknown EventType: {}", event);//TODO: print json instead
+ return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent()));
}
} catch (Exception e) {
logger.error("Failed to index entity", e, message);
- return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), false));
+ return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent()));
} finally {
messageCycle.update(System.currentTimeMillis() - event.getCreationTime());
}
});
+ //filter for success, send to the index(optional), ack
return masterObservable
//remove unsuccessful
- .filter(indexEventResult -> indexEventResult.shouldProcess())
+ .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
//take the max
.buffer( MAX_TAKE )
//map them to index results and return them
@@ -290,8 +314,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
return indexProducer.put(combined)
- .flatMap(operationResult -> Observable.from(indexEventResults))
- .map(result -> result.getQueueMessage());
+ .flatMap(indexOperationMessage -> Observable.from(indexEventResults))
+ .map(result -> result.getQueueMessage().get());
});
@@ -303,10 +327,10 @@ public class AmazonAsyncEventService implements AsyncEventService {
){
try{
return operation.call(queueMessage)
- .map(indexOperationMessage -> new IndexEventResult(queueMessage, Optional.fromNullable(indexOperationMessage), true));
+ .map(indexOperationMessage -> new IndexEventResult(Optional.fromNullable(queueMessage), Optional.fromNullable(indexOperationMessage)));
}catch (Exception e){
logger.error("failed to run index",e);
- return Observable.just( new IndexEventResult(queueMessage, Optional.<IndexOperationMessage>absent(),false));
+ return Observable.just( new IndexEventResult(Optional.fromNullable(queueMessage), Optional.<IndexOperationMessage>absent()));
}
}
@@ -543,10 +567,17 @@ public class AmazonAsyncEventService implements AsyncEventService {
//this won't block our read loop, just reads and proceeds
.flatMap(messages ->
handleMessages(messages)
- .doOnNext(message -> {
- //ack each message, but only if we didn't error.
- ack(message);
+ .buffer(MAX_TAKE)
+ .doOnNext(messagesToAck -> {
+ try {
+ //ack each message, but only if we didn't error.
+ ack(messagesToAck);
+ } catch (Exception e) {
+ logger.error("failed to ack messages to sqs", messagesToAck.get(0).getMessageId(), e);
+ //do not rethrow so we can process all of them
+ }
})
+ .flatMap(messagesToAck -> Observable.from(messagesToAck))
);
//start in the background
@@ -574,23 +605,19 @@ public class AmazonAsyncEventService implements AsyncEventService {
public class IndexEventResult{
- private final QueueMessage queueMessage;
+ private final Optional<QueueMessage> queueMessage;
private final Optional<IndexOperationMessage> indexOperationMessage;
- private final boolean shouldProcess;
- public IndexEventResult(QueueMessage queueMessage, Optional<IndexOperationMessage> indexOperationMessage ,boolean shouldProcess){
+
+ public IndexEventResult(Optional<QueueMessage> queueMessage, Optional<IndexOperationMessage> indexOperationMessage ){
this.queueMessage = queueMessage;
this.indexOperationMessage = indexOperationMessage;
- this.shouldProcess = shouldProcess;
- }
- public QueueMessage getQueueMessage() {
- return queueMessage;
}
- public boolean shouldProcess() {
- return shouldProcess;
+ public Optional<QueueMessage> getQueueMessage() {
+ return queueMessage;
}
public Optional<IndexOperationMessage> getIndexOperationMessage() {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7c5a864d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index 409c2bc..e778948 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -115,6 +115,7 @@ public class EsIndexProducerImpl implements IndexProducer {
final Observable<IndexOperation> index = Observable.from(batch.getIndexRequests());
final Observable<DeIndexOperation> deIndex = Observable.from(batch.getDeIndexRequests());
+ //TODO: look at indexing ordering
final Observable<BatchOperation> batchOps = Observable.merge(index, deIndex);
//buffer into the max size we can send ES and fire them all off until we're completed
@@ -207,9 +208,10 @@ public class EsIndexProducerImpl implements IndexProducer {
if ( error ) {
if(errorString.lastIndexOf("rejected execution (queue capacity")>=0){
try{
- log.warn("Encountered Queue Capacity Exception from ElasticSearch slowing by " +indexFig.getSleepTimeForQueueError());
+ log.warn("Encountered Queue Capacity Exception from ElasticSearch slowing by "
+ + indexFig.getSleepTimeForQueueError() );
Thread.sleep(indexFig.getSleepTimeForQueueError());
- }catch (InterruptedException ie){
+ }catch (Exception e){
//move on
}
}
[31/50] [abbrv] usergrid git commit: remove observables in favor of
streams()
Posted by sn...@apache.org.
remove observables in favor of streams()
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b4855172
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b4855172
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b4855172
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: b4855172283a62d178d157be112be345c1e150d8
Parents: 0692faa
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 8 18:07:23 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 8 18:07:23 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 95 +++++++++++---------
1 file changed, 51 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b4855172/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 5a46aed..0ac3860 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
import com.google.common.base.Optional;
import org.apache.usergrid.persistence.index.impl.IndexProducer;
@@ -255,22 +256,21 @@ public class AmazonAsyncEventService implements AsyncEventService {
logger.debug("handleMessages with {} message", messages.size());
}
- final int bufferSize = messages.size();
- Observable<IndexEventResult> masterObservable = Observable.from(messages).flatMap(message -> {
+ Observable<IndexEventResult> masterObservable = Observable.from(messages).map(message -> {
AsyncEvent event = null;
- try{
+ try {
event = (AsyncEvent) message.getBody();
- }catch (ClassCastException cce){
- logger.error("Failed to deserialize message body",cce);
+ } catch (ClassCastException cce) {
+ logger.error("Failed to deserialize message body", cce);
}
if (event == null) {
logger.error("AsyncEvent type or event is null!");
- return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(),System.currentTimeMillis()));
+ return new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(), System.currentTimeMillis());
}
final AsyncEvent thisEvent = event;
- if(logger.isDebugEnabled()) {
+ if (logger.isDebugEnabled()) {
logger.debug("Processing {} event", event);
}
@@ -287,53 +287,55 @@ public class AmazonAsyncEventService implements AsyncEventService {
indexoperationObservable = handleEntityIndexUpdate(message);
} else if (event instanceof InitializeApplicationIndexEvent) {
//does not return observable
- handleInitializeApplicationIndex(event,message);
+ handleInitializeApplicationIndex(event, message);
indexoperationObservable = Observable.just(new IndexOperationMessage());
} else {
throw new Exception("Unknown EventType");//TODO: print json instead
}
+ //collect all of the
+ IndexOperationMessage indexOperationMessage =
+ indexoperationObservable
+ .collect(() -> new IndexOperationMessage(), (collector, single ) -> collector.ingest(single))
+ .toBlocking().lastOrDefault(null);
+
//return type that can be indexed and ack'd later
- return indexoperationObservable
- .map(indexOperationMessage ->
- new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage),thisEvent.getCreationTime())
- );
+ return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
} catch (Exception e) {
logger.error("Failed to index message: " + message.getMessageId(), e, message);
- return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime()));
+ return new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime());
}
});
+ //resolve the list and return it.
+ final List<IndexEventResult> indexEventResults = masterObservable.toList().toBlocking().lastOrDefault(null);
+ //if nothing came back then return null
+ if(indexEventResults==null){
+ return null;
+ }
- //filter for success, send to the index(optional), ack
- return masterObservable
- //take the max
- .buffer(bufferSize)
- //map them to index results and return them
- .flatMap(indexEventResults -> {
- IndexOperationMessage combined = new IndexOperationMessage();
- indexEventResults.stream().forEach(
- indexEventResult -> {
- if (indexEventResult.getIndexOperationMessage().isPresent()) {
- combined.ingest(indexEventResult.getIndexOperationMessage().get());
- }
- });
-
+ final IndexOperationMessage combined = new IndexOperationMessage();
+ //stream and filer the messages
+ List<QueueMessage> messagesToAck = indexEventResults.stream()
+ .map(indexEventResult -> {
+ if (indexEventResult.getIndexOperationMessage().isPresent()) {
+ combined.ingest(indexEventResult.getIndexOperationMessage().get());
+ }
+ return indexEventResult;
+ })
+ .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
+ .map(indexEventResult -> {
+ messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime());
+ return indexEventResult;
+ })
//ack after successful completion of the operation.
- return indexProducer.put(combined)
- //change observable type
- .flatMap(indexOperationMessage -> Observable.from(indexEventResults))
- //remove unsuccessful
- .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
- //measure
- .doOnNext(indexEventResult -> messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()))
- //return the queue messages to ack
- .map(result -> result.getQueueMessage().get())
- .toList();
+ .map(result -> result.getQueueMessage().get())
+ .collect(Collectors.toList());
- })
- .doOnError(t -> logger.error("Failed to process queuemessages",t))
- .toBlocking().lastOrDefault(null);
+ //send the batch
+ indexProducer.put(combined).toBlocking().lastOrDefault(null);
+
+ return messagesToAck;
}
@@ -473,11 +475,11 @@ public class AmazonAsyncEventService implements AsyncEventService {
entityDeleteResults = eventBuilder.buildEntityDelete( applicationScope, entityId );
- final Observable<IndexOperationMessage> merged = entityDeleteResults
+ entityDeleteResults
.getEntitiesCompacted()
- .collect(() -> new ArrayList<>(), (list, item) -> list.add(item))
- .flatMap(collected -> entityDeleteResults.getIndexObservable()) ;
- return merged;
+ .collect(() -> new ArrayList<>(), (list, item) -> list.add(item)).toBlocking().lastOrDefault(null);
+
+ return entityDeleteResults.getIndexObservable();
}
@@ -574,10 +576,15 @@ public class AmazonAsyncEventService implements AsyncEventService {
List<QueueMessage> messagesToAck = handleMessages(messages);
if (messagesToAck == null || messagesToAck.size() == 0) {
+ logger.error("No messages came back from the queue operation",messages);
return messagesToAck;
}
+ if(messagesToAck.size()<messages.size()){
+ logger.error("Missing messages from queue post operation",messages,messagesToAck);
+ }
//ack each message, but only if we didn't error.
ack(messagesToAck);
+ //messagesToAck.stream().forEach(message -> ack(message));
return messagesToAck;
} catch (Exception e) {
logger.error("failed to ack messages to sqs", messages.get(0).getMessageId(), e);
[28/50] [abbrv] usergrid git commit: add additional logging
Posted by sn...@apache.org.
add additional logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0692faa1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0692faa1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0692faa1
Branch: refs/heads/asf-site
Commit: 0692faa1c2b13e9908596309e00650bf84d0f9b2
Parents: 365f6dc
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 8 15:13:27 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 8 15:13:27 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/AmazonAsyncEventService.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0692faa1/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index e3aca06..5a46aed 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -305,7 +305,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
});
//filter for success, send to the index(optional), ack
- return (List<QueueMessage>) masterObservable
+ return masterObservable
//take the max
.buffer(bufferSize)
//map them to index results and return them
@@ -328,12 +328,12 @@ public class AmazonAsyncEventService implements AsyncEventService {
//measure
.doOnNext(indexEventResult -> messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()))
//return the queue messages to ack
- .map(result -> result.getQueueMessage().get());
+ .map(result -> result.getQueueMessage().get())
+ .toList();
})
.doOnError(t -> logger.error("Failed to process queuemessages",t))
.toBlocking().lastOrDefault(null);
-
}
[45/50] [abbrv] usergrid git commit: Lowers the default viability
timeout to 10 seconds
Posted by sn...@apache.org.
Lowers the default viability timeout to 10 seconds
Changes retry count to 20. During this, we should hit the 10% read repair chance if we're missing data.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0f589f68
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0f589f68
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0f589f68
Branch: refs/heads/asf-site
Commit: 0f589f688ee72fe9db2b50e7af6604152cd898ff
Parents: dbf37e4
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Oct 14 14:27:42 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Oct 14 14:27:42 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/queue/QueueFig.java | 12 ++++++++++--
.../persistence/queue/util/AmazonNotificationUtils.java | 1 +
2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0f589f68/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
index 66f8af4..7f26bcf 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
@@ -56,10 +56,10 @@ public interface QueueFig extends GuicyFig {
String getDeadletterRetentionPeriod();
/**
- * The maximum number of messages to deliver to a dead letter queue.
+ * The maximum number of attempts to attempt to deliver before failing into the DLQ
*/
@Key( "usergrid.queue.deliveryLimit" )
- @Default("5")
+ @Default("20")
String getQueueDeliveryLimit();
@Key("usergrid.use.default.queue")
@@ -74,4 +74,12 @@ public interface QueueFig extends GuicyFig {
@Key("usergrid.queue.publish.queuesize")
@Default("850000")
int getAsyncQueueSize();
+
+ /**
+ * Set the visibility timeout for faster retries
+ * @return
+ */
+ @Key( "usergrid.queue.visibilityTimeout" )
+ @Default("10")
+ String getVisibilityTimeout();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0f589f68/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
index c321715..6105592 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
@@ -64,6 +64,7 @@ public class AmazonNotificationUtils {
final Map<String, String> queueAttributes = new HashMap<>( 2 );
queueAttributes.put( "MessageRetentionPeriod", fig.getRetentionPeriod() );
queueAttributes.put( "RedrivePolicy", redrivePolicy );
+ queueAttributes.put( "VisibilityTimeout", fig.getVisibilityTimeout() );
CreateQueueRequest createQueueRequest = new CreateQueueRequest().
withQueueName( queueName )
[35/50] [abbrv] usergrid git commit: seperate concerns so indexing
and event handling are distinct
Posted by sn...@apache.org.
seperate concerns so indexing and event handling are distinct
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/80324de9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/80324de9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/80324de9
Branch: refs/heads/asf-site
Commit: 80324de9e606ef558f129620a242ce4b0a3a25f9
Parents: 3cb0a0e
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Oct 9 08:47:25 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Oct 9 08:47:25 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 95 +++++++++++---------
1 file changed, 54 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/80324de9/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index b384b80..4ee2094 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -23,7 +23,6 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@@ -73,7 +72,6 @@ import com.google.inject.Singleton;
import rx.Observable;
import rx.Subscriber;
import rx.Subscription;
-import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -251,9 +249,14 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
- private List<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
+ /**
+ * calls the event handlers and returns a result with information on whether it needs to be ack'd and whether it needs to be indexed
+ * @param messages
+ * @return
+ */
+ private List<IndexEventResult> callEventHandlers(final List<QueueMessage> messages) {
if (logger.isDebugEnabled()) {
- logger.debug("handleMessages with {} message", messages.size());
+ logger.debug("callEventHandlers with {} message", messages.size());
}
Observable<IndexEventResult> masterObservable = Observable.from(messages).map(message -> {
@@ -310,42 +313,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
final List<IndexEventResult> indexEventResults = masterObservable
.collect(() -> new ArrayList<IndexEventResult>(), (list,indexEventResult) -> list.add(indexEventResult) )
.toBlocking().lastOrDefault(null);
- //if nothing came back then return null
- if(indexEventResults==null){
- return null;
- }
- final IndexOperationMessage combined = new IndexOperationMessage();
- //stream and filer the messages
- List<QueueMessage> messagesToAck = indexEventResults.stream()
- .map(indexEventResult -> {
- //collect into the index submission
- if (indexEventResult.getIndexOperationMessage().isPresent()) {
- combined.ingest(indexEventResult.getIndexOperationMessage().get());
- }
- return indexEventResult;
- })
- //filter out the ones that need to be ack'd
- .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
- .map(indexEventResult -> {
- //record the cycle time
- messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime());
- return indexEventResult;
- })
- //ack after successful completion of the operation.
- .map(result -> result.getQueueMessage().get())
- .collect(Collectors.toList());
-
- //send the batch
- //TODO: should retry?
- try {
- indexProducer.put(combined).toBlocking().lastOrDefault(null);
- }catch (Exception e){
- logger.error("Failed to submit to index producer",messages,e);
- throw e;
- }
- return messagesToAck;
+ return indexEventResults;
}
@@ -582,9 +552,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
try {
-
- List<QueueMessage> messagesToAck = handleMessages(messages);
-
+ List<IndexEventResult> indexEventResults = callEventHandlers(messages);
+ List<QueueMessage> messagesToAck = submitToIndex(indexEventResults);
if (messagesToAck == null || messagesToAck.size() == 0) {
logger.error("No messages came back from the queue operation",messages);
return messagesToAck;
@@ -610,6 +579,50 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
}
+ /**
+ * Submit results to index and return the queue messages to be ack'd
+ * @param indexEventResults
+ * @return
+ */
+ private List<QueueMessage> submitToIndex( List<IndexEventResult> indexEventResults) {
+ //if nothing came back then return null
+ if(indexEventResults==null){
+ return null;
+ }
+
+ final IndexOperationMessage combined = new IndexOperationMessage();
+
+ //stream and filer the messages
+ List<QueueMessage> messagesToAck = indexEventResults.stream()
+ .map(indexEventResult -> {
+ //collect into the index submission
+ if (indexEventResult.getIndexOperationMessage().isPresent()) {
+ combined.ingest(indexEventResult.getIndexOperationMessage().get());
+ }
+ return indexEventResult;
+ })
+ //filter out the ones that need to be ack'd
+ .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
+ .map(indexEventResult -> {
+ //record the cycle time
+ messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime());
+ return indexEventResult;
+ })
+ //ack after successful completion of the operation.
+ .map(result -> result.getQueueMessage().get())
+ .collect(Collectors.toList());
+
+ //send the batch
+ //TODO: should retry?
+ try {
+ indexProducer.put(combined).toBlocking().lastOrDefault(null);
+ }catch (Exception e){
+ logger.error("Failed to submit to index producer",e);
+ throw e;
+ }
+ return messagesToAck;
+ }
+
public void index(final ApplicationScope applicationScope, final Id id, final long updatedSince) {
//change to id scope to avoid serialization issues
offer( new EntityIndexEvent( new EntityIdScope( applicationScope, id ), updatedSince ) );
[44/50] [abbrv] usergrid git commit: Lowers the default viability
timeout to 10 seconds
Posted by sn...@apache.org.
Lowers the default viability timeout to 10 seconds
Changes retry count to 20. During this, we should hit the 10% read repair chance if we're missing data.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0f589f68
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0f589f68
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0f589f68
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 0f589f688ee72fe9db2b50e7af6604152cd898ff
Parents: dbf37e4
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Oct 14 14:27:42 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Oct 14 14:27:42 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/queue/QueueFig.java | 12 ++++++++++--
.../persistence/queue/util/AmazonNotificationUtils.java | 1 +
2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0f589f68/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
index 66f8af4..7f26bcf 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
@@ -56,10 +56,10 @@ public interface QueueFig extends GuicyFig {
String getDeadletterRetentionPeriod();
/**
- * The maximum number of messages to deliver to a dead letter queue.
+ * The maximum number of attempts to attempt to deliver before failing into the DLQ
*/
@Key( "usergrid.queue.deliveryLimit" )
- @Default("5")
+ @Default("20")
String getQueueDeliveryLimit();
@Key("usergrid.use.default.queue")
@@ -74,4 +74,12 @@ public interface QueueFig extends GuicyFig {
@Key("usergrid.queue.publish.queuesize")
@Default("850000")
int getAsyncQueueSize();
+
+ /**
+ * Set the visibility timeout for faster retries
+ * @return
+ */
+ @Key( "usergrid.queue.visibilityTimeout" )
+ @Default("10")
+ String getVisibilityTimeout();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0f589f68/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
index c321715..6105592 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
@@ -64,6 +64,7 @@ public class AmazonNotificationUtils {
final Map<String, String> queueAttributes = new HashMap<>( 2 );
queueAttributes.put( "MessageRetentionPeriod", fig.getRetentionPeriod() );
queueAttributes.put( "RedrivePolicy", redrivePolicy );
+ queueAttributes.put( "VisibilityTimeout", fig.getVisibilityTimeout() );
CreateQueueRequest createQueueRequest = new CreateQueueRequest().
withQueueName( queueName )
[10/50] [abbrv] usergrid git commit: move subscriber
Posted by sn...@apache.org.
move subscriber
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7dceb563
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7dceb563
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7dceb563
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 7dceb563eeb836a7dd09280b07af78c69ebb1793
Parents: 17586ec
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 6 08:39:21 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 6 08:39:21 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 5 ++-
.../core/rx/ExceptionBehaviorTest.java | 32 +++++++++++++++++++-
2 files changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7dceb563/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index e16de05..14d37b5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -547,12 +547,11 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack each message, but only if we didn't error.
ack(message);
})
- )
- .subscribeOn(Schedulers.newThread());
+ );
//start in the background
- final Subscription subscription = consumer.subscribe();
+ final Subscription subscription = consumer.subscribeOn(Schedulers.newThread()).subscribe();
subscriptions.add(subscription);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7dceb563/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
index be10d0a..8e4f4c4 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
@@ -22,6 +22,7 @@ import org.junit.Test;
import rx.Observable;
import rx.Observer;
+import rx.schedulers.Schedulers;
/**
@@ -31,13 +32,21 @@ public class ExceptionBehaviorTest {
//this test shows toBlocking re-throws exceptions correctly
@Test( expected = TestException.class )
- public void throwOnBlocking() {
+ public void throwOnBlockingFirst() {
Observable.range( 0, 1 ).map( integer -> {
throw new TestException( "I throw and exception" );
} ).toBlocking().first();
}
+ @Test( expected = TestException.class )
+ public void throwOnBlockingLast() {
+
+ Observable.range( 0, 1 ).map( integer -> {
+ throw new TestException( "I throw and exception" );
+ } ).toBlocking().last();
+ }
+
//
// /**
// * This shows that no re-throw happens on subscribe. This is as designed, but not as expected
@@ -67,6 +76,27 @@ public class ExceptionBehaviorTest {
exceptionObserver.checkResult();
}
+ /**
+ * Tests working with observers
+ */
+ @Test( expected = TestException.class )
+ public void throwOnSubscribeObservableNewThread() throws Exception {
+
+ final ReThrowObserver exceptionObserver = new ReThrowObserver();
+
+ Observable.range( 0, 1 ).map(integer -> {
+ throw new TestException("I throw and exception");
+ })
+ .doOnError(t -> exceptionObserver.onError(t))
+ .subscribeOn(Schedulers.newThread())
+ .subscribe(exceptionObserver);
+
+ for(int i =0; i<5; i++) {
+ exceptionObserver.checkResult();
+ Thread.sleep(200);
+ }
+ }
+
private static final class TestException extends RuntimeException {
[29/50] [abbrv] usergrid git commit: add additional logging
Posted by sn...@apache.org.
add additional logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0692faa1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0692faa1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0692faa1
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 0692faa1c2b13e9908596309e00650bf84d0f9b2
Parents: 365f6dc
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 8 15:13:27 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 8 15:13:27 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/AmazonAsyncEventService.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0692faa1/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index e3aca06..5a46aed 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -305,7 +305,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
});
//filter for success, send to the index(optional), ack
- return (List<QueueMessage>) masterObservable
+ return masterObservable
//take the max
.buffer(bufferSize)
//map them to index results and return them
@@ -328,12 +328,12 @@ public class AmazonAsyncEventService implements AsyncEventService {
//measure
.doOnNext(indexEventResult -> messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()))
//return the queue messages to ack
- .map(result -> result.getQueueMessage().get());
+ .map(result -> result.getQueueMessage().get())
+ .toList();
})
.doOnError(t -> logger.error("Failed to process queuemessages",t))
.toBlocking().lastOrDefault(null);
-
}
[23/50] [abbrv] usergrid git commit: buffer size fix
Posted by sn...@apache.org.
buffer size fix
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f9f08253
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f9f08253
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f9f08253
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: f9f0825315c22ecfa1375189bf70020cde868661
Parents: 79caa09
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 7 15:14:22 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 7 15:14:22 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f9f08253/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index a5342ea..a9e2459 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -255,6 +255,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
logger.debug("handleMessages with {} message", messages.size());
}
+ final int bufferSize = messages.size();
Observable<IndexEventResult> masterObservable = Observable.from(messages).flatMap(message -> {
AsyncEvent event = null;
try{
@@ -305,10 +306,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
//filter for success, send to the index(optional), ack
return masterObservable
- //remove unsuccessful
- .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
//take the max
- .buffer( MAX_TAKE )
+ .buffer(250, TimeUnit.MILLISECONDS, bufferSize)
//map them to index results and return them
.flatMap(indexEventResults -> {
IndexOperationMessage combined = new IndexOperationMessage();
@@ -322,8 +321,13 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
return indexProducer.put(combined)
+ //change observable type
.flatMap(indexOperationMessage -> Observable.from(indexEventResults))
+ //remove unsuccessful
+ .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
+ //measure
.doOnNext(indexEventResult -> messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()))
+ //return the queue messages to ack
.map(result -> result.getQueueMessage().get());
});
[16/50] [abbrv] usergrid git commit: remove integration tests
Posted by sn...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/equals.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/equals.js b/stack/rest_integration_tests/test/queries/equals.js
deleted file mode 100644
index 904646b..0000000
--- a/stack/rest_integration_tests/test/queries/equals.js
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("filter " + config.genericTestCollection1 + " with '=' and '!=' queries", function(done) {
- var query1 = "where consistentProperty = 'somethingConsistent'";
- maxNumberOfEntities = Math.max(config.numberOfEntities, 100);
- it('should return ' + config.numberOfEntities + ' results ' + query1, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query1, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.equal(config.numberOfEntities);
- body.entities.forEach(function(entity) {
- entity.consistentProperty.should.equal('somethingConsistent');
- });
- done();
- });
- });
-
- var query2 = "where title = 'A Tale of Two Cities'";
- maxNumberOfEntities = Math.max(config.numberOfEntities, 100);
- it('should return ' + config.numberOfEntities + ' results ' + query2, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query2, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.equal(config.numberOfEntities);
- body.entities.forEach(function(entity) {
- entity.title.should.equal('A Tale of Two Cities');
- });
- done();
- });
- });
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/location.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/location.js b/stack/rest_integration_tests/test/queries/location.js
deleted file mode 100644
index 6fbc96a..0000000
--- a/stack/rest_integration_tests/test/queries/location.js
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-var response = require('../../lib/response');
-
-module.exports = {
- test: function() {
- describe("filter " + config.genericTestCollection2 + " with location queries", function(done) {
- var locationString = config.location.latitude + ", " + config.location.longitude;
- var query = "location within 1000 of " + locationString;
- maxNumberOfEntities = Math.max(config.numberOfEntities, 100);
- it("should return all results with a location within 1000m of " + locationString, function(done) {
- entities.getWithQuery(config.genericTestCollection2, query, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.forEach(function(entity) {
- var distance = response.distanceInMeters(config.location, entity.location);
- distance.should.be.lessThan(1000);
- });
- done();
- });
- });
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/order.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/order.js b/stack/rest_integration_tests/test/queries/order.js
deleted file mode 100644
index 8cbb78f..0000000
--- a/stack/rest_integration_tests/test/queries/order.js
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("sort " + config.genericTestCollection1 + " with 'order by' queries", function(done) {
- var query1 = "order by created desc";
- it('should return a subset of results ' + query1.replace('order', 'ordered'), function(done) {
- entities.getWithQuery(config.genericTestCollection1, query1, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- var comparisonArray = body.entities;
- comparisonArray.sort(function(a, b) {
- return a.created + b.created;
- });
- body.entities.should.equal(comparisonArray);
- done();
- });
- });
- var query2 = "order by created asc";
- it('should return a subset of results ' + query2.replace('order', 'ordered'), function(done) {
- entities.getWithQuery(config.genericTestCollection1, query2, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- var comparisonArray = body.entities;
- comparisonArray.sort(function(a, b) {
- return a.created - b.created;
- });
- body.entities.should.equal(comparisonArray);
- done();
- });
- });
- var query3 = "order by optionsProperty desc";
- it('should return a subset of results ' + query3.replace('order', 'ordered'), function(done) {
- entities.getWithQuery(config.genericTestCollection1, query3, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- var comparisonArray = body.entities;
- comparisonArray.sort(function(a, b) {
- if (a.optionsProperty > b.optionsProperty) return -1;
- if (a.optionsProperty < b.optionsProperty) return 1;
- return 0;
- });
- body.entities.should.equal(comparisonArray);
- done();
- });
- });
- var query4 = "order by optionsProperty asc";
- it('should return a subset of results ' + query4.replace('order', 'ordered'), function(done) {
- entities.getWithQuery(config.genericTestCollection1, query4, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- var comparisonArray = body.entities;
- comparisonArray.sort(function(a, b) {
- if (a.optionsProperty < b.optionsProperty) return -1;
- if (a.optionsProperty > b.optionsProperty) return 1;
- return 0;
- });
- done();
- });
- });
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/setup.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/setup.js b/stack/rest_integration_tests/test/setup.js
deleted file mode 100644
index 4fbc664..0000000
--- a/stack/rest_integration_tests/test/setup.js
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-// here we're going to do teardown of BaaS environment - deletion of users, entities, etc.
-var users = require("../lib/users");
-var entities = require("../lib/entities");
-var config = require("../config");
-var async = require("async");
-var uuid = require("uuid");
-var should = require("should");
-
-module.exports = {
- users: [],
- admins: [],
- do: function(cb) {
- console.log(" setup");
- async.parallel([
- function(cb) {
- // create admin user
- var id = uuid.v1().toString().replace("-", "");
- var admin = {
- username: id + "user",
- password: "pwd" + id,
- name: id + " name",
- email: id + "@uge2e.com"
- }
- users.add(admin, function(err, user) {
- users.addToRole(user.username, "admin", function(err) {
- module.exports.admins.push(admin);
- cb(err, err ? null : admin);
- });
- });
- },
- function(cb) {
- // create app users
- var size = config.numberOfUsers;
- var userArr = [];
- module.exports.users = userArr;
- for (var i = 0; i < size; i++) {
- var id = uuid.v1().toString().replace("-", "");
- userArr.push({
- username: id + "user",
- password: "pwd" + id,
- name: id + " name",
- email: id + "@uge2e.com"
- });
- }
- async.each(
- userArr,
- function(user, cb) {
- users.add(user, function(err, user) {
- cb(err, user);
- });
- },
- function(err, localUsers) {
- cb(err);
- }
- )
- },
- function(cb) {
- // create entities
- async.parallel([
- function(cb) {
- entities.create(config.genericTestCollection1, config.numberOfEntities, function(err, body) {
- cb(err);
- });
- },
- function(cb) {
- entities.create(config.genericTestCollection2, config.numberOfEntities, function(err, body) {
- cb(err);
- });
- },
- function(cb) {
- entities.create(config.consumableTestCollection, config.numberOfEntities, function(err, body) {
- cb(err);
- });
- }
- ],
- function(err, data) {
- cb(err);
- });
- }
- ],
- function(err, data) {
- should(err).be.null;
- console.log(" ✓".green + " done".grey);
- if (cb && typeof(cb) === 'function') cb(err);
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/teardown.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/teardown.js b/stack/rest_integration_tests/test/teardown.js
deleted file mode 100644
index 7afdc97..0000000
--- a/stack/rest_integration_tests/test/teardown.js
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../lib/entities");
-var should = require("should");
-var async = require("async");
-var config = require("../config");
-var inflect = require("i")();
-
-module.exports = {
- do: function(cb) {
- console.log(" teardown");
- async.parallel([
- function(cb) {
- entities.deleteAll(config.usersCollection, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- cb(err);
- })
- },
- function(cb) {
- entities.deleteAll(config.genericTestCollection1, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- cb(err);
- })
- },
- function(cb) {
- entities.deleteAll(config.genericTestCollection2, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- cb(err);
- })
- },
- function(cb) {
- entities.deleteAll(inflect.pluralize(config.consumableTestCollection), function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- cb(err);
- })
- }
- ],
- function(err, data) {
- console.log(" ✓".green + " done".grey);
- if (cb && typeof(cb) === 'function') cb(err);
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/users/create.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/users/create.js b/stack/rest_integration_tests/test/users/create.js
deleted file mode 100644
index c62ccc4..0000000
--- a/stack/rest_integration_tests/test/users/create.js
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var should = require("should");
-var uuid = require("uuid");
-var users = require("../../lib/users");
-
-module.exports = {
- test: function() {
- describe("create a user", function() {
- var username = "testuser" + uuid.v1();
- var password = "password";
- it("should successfully return a user object", function(done) {
- users.add({
- username: username,
- password: password,
- name: username + " name",
- email: username + "@uge2e.com"
- }, function(err, userBody) {
- should(err).be.null;
- userBody.should.not.be.null;
- users.login(username, password, function(err, body) {
- should(err).be.null;
- body.should.not.be.null;
- body.should.have.property("access_token");
- done();
- })
- });
- });
- });
- }
-}
[50/50] [abbrv] usergrid git commit: Renaming to work around case
issue.
Posted by sn...@apache.org.
Renaming to work around case issue.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ec2c4478
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ec2c4478
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ec2c4478
Branch: refs/heads/asf-site
Commit: ec2c4478875a8d7be4af3c770c3523d43e656e4f
Parents: d68f450
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 15 10:15:52 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 15 10:15:52 2015 -0400
----------------------------------------------------------------------
website/content/img/mike_d.jpg | Bin 0 -> 36443 bytes
website/content/img/miked.jpg | Bin 36443 -> 0 bytes
2 files changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ec2c4478/website/content/img/mike_d.jpg
----------------------------------------------------------------------
diff --git a/website/content/img/mike_d.jpg b/website/content/img/mike_d.jpg
new file mode 100644
index 0000000..8dca2ee
Binary files /dev/null and b/website/content/img/mike_d.jpg differ
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ec2c4478/website/content/img/miked.jpg
----------------------------------------------------------------------
diff --git a/website/content/img/miked.jpg b/website/content/img/miked.jpg
deleted file mode 100644
index 8dca2ee..0000000
Binary files a/website/content/img/miked.jpg and /dev/null differ
[36/50] [abbrv] usergrid git commit: add a better status code check
for index overload
Posted by sn...@apache.org.
add a better status code check for index overload
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5b650860
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5b650860
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5b650860
Branch: refs/heads/asf-site
Commit: 5b6508609c3ce1687993b4015f88126c8be6676d
Parents: 80324de
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Oct 9 11:34:16 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Oct 9 11:34:16 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EsIndexProducerImpl.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5b650860/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index e778948..031556a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -27,6 +27,7 @@ import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.Client;
+import org.elasticsearch.rest.RestStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -192,13 +193,17 @@ public class EsIndexProducerImpl implements IndexProducer {
final StringBuilder errorString = new StringBuilder( );
+ boolean hasTooManyRequests= false;
for ( BulkItemResponse response : responses ) {
if ( response.isFailed() ) {
// log error and continue processing
log.error( "Unable to index id={}, type={}, index={}, failureMessage={} ", response.getId(),
- response.getType(), response.getIndex(), response.getFailureMessage() );
-
+ response.getType(), response.getIndex(), response.getFailureMessage() );
+ //if index is overloaded on the queue fail.
+ if(response.getFailure()!=null && response.getFailure().getStatus() == RestStatus.TOO_MANY_REQUESTS){
+ hasTooManyRequests =true;
+ }
error = true;
errorString.append( response.getFailureMessage() ).append( "\n" );
@@ -206,7 +211,7 @@ public class EsIndexProducerImpl implements IndexProducer {
}
if ( error ) {
- if(errorString.lastIndexOf("rejected execution (queue capacity")>=0){
+ if(hasTooManyRequests){
try{
log.warn("Encountered Queue Capacity Exception from ElasticSearch slowing by "
+ indexFig.getSleepTimeForQueueError() );
[47/50] [abbrv] usergrid git commit: merge 2.1-release
Posted by sn...@apache.org.
merge 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f19ae087
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f19ae087
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f19ae087
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: f19ae0870ac86fba312f5604d9c2a1474de65a88
Parents: 91fa489 0f589f6
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 14 15:50:07 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 14 15:50:07 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 22 +-
.../asyncevents/AmazonAsyncEventService.java | 280 ++++++++++++-------
.../asyncevents/EventBuilderImpl.java | 3 +-
.../asyncevents/InMemoryAsyncEventService.java | 7 +-
.../index/IndexProcessorFig.java | 4 +
.../index/ReIndexServiceImpl.java | 2 +-
.../migration/DeDupConnectionDataMigration.java | 2 +-
.../read/search/CandidateEntityFilter.java | 3 +-
.../pipeline/read/search/CandidateIdFilter.java | 2 +-
.../core/rx/ExceptionBehaviorTest.java | 171 +++++++++++
.../usergrid/persistence/index/IndexFig.java | 3 +
.../index/impl/EsIndexProducerImpl.java | 28 +-
.../EsIndexMappingMigrationPlugin.java | 2 +-
.../usergrid/persistence/queue/QueueFig.java | 12 +-
.../persistence/queue/QueueMessage.java | 10 +
.../queue/impl/SNSQueueManagerImpl.java | 2 +
.../queue/impl/SQSQueueManagerImpl.java | 1 +
.../queue/util/AmazonNotificationUtils.java | 1 +
.../rest/system/ApplicationsResource.java | 28 +-
stack/rest_integration_tests/README.md | 19 --
stack/rest_integration_tests/config/default.js | 38 ---
stack/rest_integration_tests/config/index.js | 25 --
stack/rest_integration_tests/index.js | 20 --
stack/rest_integration_tests/lib/connections.js | 137 ---------
stack/rest_integration_tests/lib/entities.js | 166 -----------
.../rest_integration_tests/lib/notifications.js | 68 -----
stack/rest_integration_tests/lib/random.js | 74 -----
stack/rest_integration_tests/lib/response.js | 30 --
stack/rest_integration_tests/lib/token.js | 49 ----
stack/rest_integration_tests/lib/urls.js | 37 ---
stack/rest_integration_tests/lib/users.js | 82 ------
stack/rest_integration_tests/package.json | 18 --
.../test/authentication/management.js | 38 ---
.../test/authentication/org.js | 37 ---
.../test/authentication/resetPassword.js | 78 ------
.../test/authentication/user.js | 40 ---
.../test/connections/create.js | 49 ----
.../test/connections/delete.js | 77 -----
.../test/connections/get.js | 82 ------
.../test/entities/create.js | 60 ----
.../test/entities/deleteAll.js | 37 ---
.../rest_integration_tests/test/entities/get.js | 51 ----
.../test/entities/update.js | 43 ---
stack/rest_integration_tests/test/main.js | 72 -----
stack/rest_integration_tests/test/mocha.opts | 3 -
.../test/notifications/create.js | 36 ---
.../test/queries/comparison.js | 58 ----
.../test/queries/contains.js | 116 --------
.../test/queries/equals.js | 54 ----
.../test/queries/location.js | 42 ---
.../test/queries/order.js | 82 ------
stack/rest_integration_tests/test/setup.js | 103 -------
stack/rest_integration_tests/test/teardown.js | 65 -----
.../rest_integration_tests/test/users/create.js | 45 ---
.../management/AppInfoMigrationPlugin.java | 2 +-
.../services/AbstractConnectionsService.java | 2 +-
56 files changed, 452 insertions(+), 2166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index f1e6ad6,aad7610..1bd5c7d
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -525,11 -527,16 +526,12 @@@ public class CpRelationManager implemen
*
*/
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
- final EntityIndexBatch batch = ei.createBatch();
-
- // remove item from collection index
- SearchEdge indexScope = createCollectionSearchEdge( cpHeadEntity.getId(), collectionName );
- batch.deindex( indexScope, memberEntity );
+ //TODO: this should not happen here, needs to go to SQS
+ //indexProducer.put(batch).subscribe();
+ indexService.queueEntityDelete(applicationScope,memberEntity.getId());
- managerCache.getIndexProducer().put( batch.build()).toBlocking().lastOrDefault(null); // this should throw an exception
+
// special handling for roles collection of a group
if ( headEntity.getType().equals( Group.ENTITY_TYPE ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
index 82bc4c8,0000000..55887c7
mode 100644,000000..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
@@@ -1,168 -1,0 +1,178 @@@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.rest.system;
+
+import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Classy class class.
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class ApplicationsResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger(ApplicationsResource.class);
+
+
+ public ApplicationsResource() {
+ logger.info( "ApplicationsResource initialized" );
+ }
+
+ @RequireSystemAccess
+ @DELETE
+ @JSONP
+ @Path( "{applicationId}" )
+ public ApiResponse clearApplication(
+ @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @QueryParam( "confirmApplicationName" ) String confirmApplicationName,
+ @QueryParam( "limit" ) int limit,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+
+ throws Exception {
+
+ if(confirmApplicationName == null){
+ throw new IllegalArgumentException("please make add a QueryString for confirmApplicationName");
+ }
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final EntityManager em = emf.getEntityManager(applicationId);
+ final String name = em.getApplication().getApplicationName();
+ if(!name.toLowerCase().equals(confirmApplicationName.toLowerCase())){
+ throw new IllegalArgumentException(
+ "confirmApplicationName: " + confirmApplicationName + " does not equal " + name);
+ }
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ logger.info("clearing up application");
+
+ final Thread delete = new Thread() {
+
+ @Override
+ public void run() {
+ final AtomicInteger itemsDeleted = new AtomicInteger(0);
+ try {
+ management.deleteAllEntities(applicationId, limit)
+ .map(id -> itemsDeleted.incrementAndGet())
+ .doOnNext(count -> {
+ if( count % 100 == 0 ){
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("count",itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map).subscribe();
++ statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map)
++ .subscribe();//do not want to throw this exception
+ }
+ })
- .doOnCompleted(() ->{
- Map<String,Object> map = new LinkedHashMap<>();
- map.put("count",itemsDeleted.intValue());
++ .doOnCompleted(() -> {
++ Map<String, Object> map = new LinkedHashMap<>();
++ map.put("count", itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId,jobId, StatusService.Status.COMPLETE,map).subscribe();
++ statusService.setStatus(applicationId, jobId, StatusService.Status.COMPLETE, map)
++ .toBlocking().lastOrDefault(null);//want to rethrow this exception
+ })
- .subscribe();
++ .toBlocking().lastOrDefault(null);//expecting exception to be caught if job fails
+
+ } catch ( Exception e ) {
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("exception",e);
- statusService.setStatus(applicationId,jobId, StatusService.Status.FAILED,map).subscribe();
++ try {
++ statusService.setStatus(applicationId, jobId, StatusService.Status.FAILED, map).toBlocking().lastOrDefault(null);//leave as subscribe if fails retry
++ }catch (Exception subE){
++ logger.error("failed to update status "+jobId,subE);
++ }
+ logger.error( "Failed to delete appid:"+applicationId + " jobid:"+jobId+" count:"+itemsDeleted, e );
+ }
+ }
+ };
+
+ delete.setName("Delete for app : " + applicationId + " job: " + jobId);
+ delete.setDaemon(true);
+ delete.start();
+
- statusService.setStatus(applicationId,jobId, StatusService.Status.STARTED,new LinkedHashMap<>()).subscribe();
-
++ try {
++ //should throw exception if can't start
++ statusService.setStatus(applicationId, jobId, StatusService.Status.STARTED, new LinkedHashMap<>()).toBlocking().lastOrDefault(null);
++ }catch (Exception e){
++ logger.error("failed to set status for " + jobId, e);
++ }
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put("status",StatusService.Status.STARTED);
+ response.setData(data);
+ response.setSuccess();
+ return response;
+ }
+
+ @RequireSystemAccess
+ @GET
+ @Path( "{applicationId}/job/{jobId}" )
+ public ApiResponse getStatus(
+ @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @PathParam("jobId") UUID jobId,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception{
+
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ StatusService.JobStatus jobStatus = statusService.getStatus(applicationId, jobId).toBlocking().lastOrDefault(null);
+
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put( "status", jobStatus.getStatus().toString() );
+ data.put( "metadata", jobStatus.getData() );
+ response.setData(data);
+ response.setSuccess();
+ return response;
+ }
+
+}
[34/50] [abbrv] usergrid git commit: seperate concerns so indexing
and event handling are distinct
Posted by sn...@apache.org.
seperate concerns so indexing and event handling are distinct
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/80324de9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/80324de9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/80324de9
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 80324de9e606ef558f129620a242ce4b0a3a25f9
Parents: 3cb0a0e
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Oct 9 08:47:25 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Oct 9 08:47:25 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 95 +++++++++++---------
1 file changed, 54 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/80324de9/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index b384b80..4ee2094 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -23,7 +23,6 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@@ -73,7 +72,6 @@ import com.google.inject.Singleton;
import rx.Observable;
import rx.Subscriber;
import rx.Subscription;
-import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -251,9 +249,14 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
- private List<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
+ /**
+ * calls the event handlers and returns a result with information on whether it needs to be ack'd and whether it needs to be indexed
+ * @param messages
+ * @return
+ */
+ private List<IndexEventResult> callEventHandlers(final List<QueueMessage> messages) {
if (logger.isDebugEnabled()) {
- logger.debug("handleMessages with {} message", messages.size());
+ logger.debug("callEventHandlers with {} message", messages.size());
}
Observable<IndexEventResult> masterObservable = Observable.from(messages).map(message -> {
@@ -310,42 +313,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
final List<IndexEventResult> indexEventResults = masterObservable
.collect(() -> new ArrayList<IndexEventResult>(), (list,indexEventResult) -> list.add(indexEventResult) )
.toBlocking().lastOrDefault(null);
- //if nothing came back then return null
- if(indexEventResults==null){
- return null;
- }
- final IndexOperationMessage combined = new IndexOperationMessage();
- //stream and filer the messages
- List<QueueMessage> messagesToAck = indexEventResults.stream()
- .map(indexEventResult -> {
- //collect into the index submission
- if (indexEventResult.getIndexOperationMessage().isPresent()) {
- combined.ingest(indexEventResult.getIndexOperationMessage().get());
- }
- return indexEventResult;
- })
- //filter out the ones that need to be ack'd
- .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
- .map(indexEventResult -> {
- //record the cycle time
- messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime());
- return indexEventResult;
- })
- //ack after successful completion of the operation.
- .map(result -> result.getQueueMessage().get())
- .collect(Collectors.toList());
-
- //send the batch
- //TODO: should retry?
- try {
- indexProducer.put(combined).toBlocking().lastOrDefault(null);
- }catch (Exception e){
- logger.error("Failed to submit to index producer",messages,e);
- throw e;
- }
- return messagesToAck;
+ return indexEventResults;
}
@@ -582,9 +552,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
try {
-
- List<QueueMessage> messagesToAck = handleMessages(messages);
-
+ List<IndexEventResult> indexEventResults = callEventHandlers(messages);
+ List<QueueMessage> messagesToAck = submitToIndex(indexEventResults);
if (messagesToAck == null || messagesToAck.size() == 0) {
logger.error("No messages came back from the queue operation",messages);
return messagesToAck;
@@ -610,6 +579,50 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
}
+ /**
+ * Submit results to index and return the queue messages to be ack'd
+ * @param indexEventResults
+ * @return
+ */
+ private List<QueueMessage> submitToIndex( List<IndexEventResult> indexEventResults) {
+ //if nothing came back then return null
+ if(indexEventResults==null){
+ return null;
+ }
+
+ final IndexOperationMessage combined = new IndexOperationMessage();
+
+ //stream and filer the messages
+ List<QueueMessage> messagesToAck = indexEventResults.stream()
+ .map(indexEventResult -> {
+ //collect into the index submission
+ if (indexEventResult.getIndexOperationMessage().isPresent()) {
+ combined.ingest(indexEventResult.getIndexOperationMessage().get());
+ }
+ return indexEventResult;
+ })
+ //filter out the ones that need to be ack'd
+ .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
+ .map(indexEventResult -> {
+ //record the cycle time
+ messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime());
+ return indexEventResult;
+ })
+ //ack after successful completion of the operation.
+ .map(result -> result.getQueueMessage().get())
+ .collect(Collectors.toList());
+
+ //send the batch
+ //TODO: should retry?
+ try {
+ indexProducer.put(combined).toBlocking().lastOrDefault(null);
+ }catch (Exception e){
+ logger.error("Failed to submit to index producer",e);
+ throw e;
+ }
+ return messagesToAck;
+ }
+
public void index(final ApplicationScope applicationScope, final Id id, final long updatedSince) {
//change to id scope to avoid serialization issues
offer( new EntityIndexEvent( new EntityIdScope( applicationScope, id ), updatedSince ) );
[40/50] [abbrv] usergrid git commit: add better logging for empty
sequences
Posted by sn...@apache.org.
add better logging for empty sequences
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9cbe283d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9cbe283d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9cbe283d
Branch: refs/heads/asf-site
Commit: 9cbe283d0af45b877fb93dfae81cdb343cec26c5
Parents: a2a07aa
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 12 16:27:13 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 12 16:27:13 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 13 ++++++-----
.../asyncevents/EventBuilderImpl.java | 3 ++-
.../core/rx/ExceptionBehaviorTest.java | 24 ++++++++++++++++++++
.../persistence/queue/QueueMessage.java | 10 ++++++++
.../queue/impl/SNSQueueManagerImpl.java | 2 ++
.../queue/impl/SQSQueueManagerImpl.java | 1 +
6 files changed, 46 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 45d83cd..f1a02ce 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -299,17 +299,18 @@ public class AmazonAsyncEventService implements AsyncEventService {
//collect all of the
IndexOperationMessage indexOperationMessage =
indexoperationObservable
- .collect(() -> new IndexOperationMessage(), (collector, single ) -> collector.ingest(single))
+ .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
.toBlocking().lastOrDefault(null);
- if(indexOperationMessage == null){
- throw new IllegalArgumentException("Received null index operation.");
+ if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
+ logger.info("Received empty index sequence message:({}), body:({}) ",
+ message.getMessageId(),message.getStringBody());
}
//return type that can be indexed and ack'd later
return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
} catch (Exception e) {
- logger.error("Failed to index message: " + message.getMessageId(), e, message);
+ logger.error("Failed to index message: " + message.getMessageId(), message.getStringBody() ,e);
return new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime());
}
});
@@ -559,7 +560,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
List<IndexEventResult> indexEventResults = callEventHandlers(messages);
List<QueueMessage> messagesToAck = submitToIndex(indexEventResults);
if (messagesToAck == null || messagesToAck.size() == 0) {
- logger.error("No messages came back from the queue operation",messages);
+ logger.error("No messages came back from the queue operation should have seen "+messages.size(),messages);
return messagesToAck;
}
if(messagesToAck.size()<messages.size()){
@@ -569,7 +570,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
ack(messagesToAck);
return messagesToAck;
} catch (Exception e) {
- logger.error("failed to ack messages to sqs", messages, e);
+ logger.error("failed to ack messages to sqs", e);
return null;
//do not rethrow so we can process all of them
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 4bf5695..cc0356b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -181,8 +181,9 @@ public class EventBuilderImpl implements EventBuilder {
return true;
}
+ //entityIndexOperation.getUpdatedSince will always be 0 except for reindexing the application
//only re-index if it has been updated and been updated after our timestamp
- return modified.getValue() >= entityIndexOperation.getUpdatedSince();
+ return modified.getValue() >= entityIndexOperation.getUpdatedSince();
} )
//perform indexing on the task scheduler and start it
.flatMap( entity -> indexService.indexEntity( applicationScope, entity ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
index 8e4f4c4..cb39ca1 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
@@ -18,12 +18,16 @@
package org.apache.usergrid.persistence.core.rx;
+import org.junit.Assert;
import org.junit.Test;
import rx.Observable;
import rx.Observer;
import rx.schedulers.Schedulers;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* Tests RX exception behavior
@@ -47,6 +51,26 @@ public class ExceptionBehaviorTest {
} ).toBlocking().last();
}
+ @Test()
+ public void testSequence(){
+ ArrayList listReturn = Observable.range(0, 1).flatMap(i -> Observable.empty())
+ .collect(()->new ArrayList(),(list,i) ->{
+ list.add(i);
+ }).toBlocking().lastOrDefault(null);
+
+ Assert.assertEquals(listReturn,new ArrayList<Integer>());
+ }
+
+ @Test()
+ public void testSequence2(){
+ ArrayList listReturn = Observable.range(0, 2).buffer(2).flatMap(i -> Observable.empty())
+ .collect(()->new ArrayList(),(list,i) ->{
+ list.add(i);
+ }).toBlocking().lastOrDefault(null);
+
+ Assert.assertEquals(listReturn,new ArrayList<Integer>());
+ }
+
//
// /**
// * This shows that no re-throw happens on subscribe. This is as designed, but not as expected
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java
index 0874e9c..55f79f4 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueMessage.java
@@ -22,6 +22,7 @@ public class QueueMessage {
private final String messageId;
private final String handle;
private final String type;
+ private String stringBody;
public QueueMessage(String messageId, String handle, Object body,String type) {
@@ -29,6 +30,7 @@ public class QueueMessage {
this.messageId = messageId;
this.handle = handle;
this.type = type;
+ this.stringBody = "";
}
public String getHandle() {
@@ -47,4 +49,12 @@ public class QueueMessage {
public String getType() {
return type;
}
+
+ public void setStringBody(String stringBody) {
+ this.stringBody = stringBody;
+ }
+
+ public String getStringBody() {
+ return stringBody;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index bc63f53..a2b5d72 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -364,6 +364,7 @@ public class SNSQueueManagerImpl implements QueueManager {
for (Message message : messages) {
Object body;
+ final String originalBody = message.getBody();
try {
final JsonNode bodyNode = mapper.readTree(message.getBody());
@@ -375,6 +376,7 @@ public class SNSQueueManagerImpl implements QueueManager {
}
QueueMessage queueMessage = new QueueMessage(message.getMessageId(), message.getReceiptHandle(), body, message.getAttributes().get("type"));
+ queueMessage.setStringBody(originalBody);
queueMessages.add(queueMessage);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9cbe283d/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
index daa1cb5..fa9a7ac 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SQSQueueManagerImpl.java
@@ -196,6 +196,7 @@ public class SQSQueueManagerImpl implements QueueManager {
}
QueueMessage queueMessage = new QueueMessage(message.getMessageId(), message.getReceiptHandle(), body, message.getAttributes().get("type"));
+ queueMessage.setStringBody(message.getBody());
queueMessages.add(queueMessage);
}
[49/50] [abbrv] usergrid git commit: Renaming to work around case
issue.
Posted by sn...@apache.org.
Renaming to work around case issue.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d68f4508
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d68f4508
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d68f4508
Branch: refs/heads/asf-site
Commit: d68f4508b08063e18c735ee63363e8cb102c3f81
Parents: f19ae08
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 15 10:15:10 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 15 10:15:10 2015 -0400
----------------------------------------------------------------------
website/content/img/mike_d.JPG | Bin 36443 -> 0 bytes
website/content/img/miked.jpg | Bin 0 -> 36443 bytes
2 files changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d68f4508/website/content/img/mike_d.JPG
----------------------------------------------------------------------
diff --git a/website/content/img/mike_d.JPG b/website/content/img/mike_d.JPG
deleted file mode 100644
index 8dca2ee..0000000
Binary files a/website/content/img/mike_d.JPG and /dev/null differ
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d68f4508/website/content/img/miked.jpg
----------------------------------------------------------------------
diff --git a/website/content/img/miked.jpg b/website/content/img/miked.jpg
new file mode 100644
index 0000000..8dca2ee
Binary files /dev/null and b/website/content/img/miked.jpg differ
[37/50] [abbrv] usergrid git commit: add a better status code check
for index overload
Posted by sn...@apache.org.
add a better status code check for index overload
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5b650860
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5b650860
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5b650860
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 5b6508609c3ce1687993b4015f88126c8be6676d
Parents: 80324de
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Oct 9 11:34:16 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Oct 9 11:34:16 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EsIndexProducerImpl.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5b650860/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index e778948..031556a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -27,6 +27,7 @@ import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.client.Client;
+import org.elasticsearch.rest.RestStatus;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -192,13 +193,17 @@ public class EsIndexProducerImpl implements IndexProducer {
final StringBuilder errorString = new StringBuilder( );
+ boolean hasTooManyRequests= false;
for ( BulkItemResponse response : responses ) {
if ( response.isFailed() ) {
// log error and continue processing
log.error( "Unable to index id={}, type={}, index={}, failureMessage={} ", response.getId(),
- response.getType(), response.getIndex(), response.getFailureMessage() );
-
+ response.getType(), response.getIndex(), response.getFailureMessage() );
+ //if index is overloaded on the queue fail.
+ if(response.getFailure()!=null && response.getFailure().getStatus() == RestStatus.TOO_MANY_REQUESTS){
+ hasTooManyRequests =true;
+ }
error = true;
errorString.append( response.getFailureMessage() ).append( "\n" );
@@ -206,7 +211,7 @@ public class EsIndexProducerImpl implements IndexProducer {
}
if ( error ) {
- if(errorString.lastIndexOf("rejected execution (queue capacity")>=0){
+ if(hasTooManyRequests){
try{
log.warn("Encountered Queue Capacity Exception from ElasticSearch slowing by "
+ indexFig.getSleepTimeForQueueError() );
[27/50] [abbrv] usergrid git commit: add additional logging
Posted by sn...@apache.org.
add additional logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/365f6dc2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/365f6dc2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/365f6dc2
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 365f6dc2ac36720b221d043ef0c28503c2a9513c
Parents: d57b4fe
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 8 14:39:15 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 8 14:39:15 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 52 +++++++++++---------
1 file changed, 28 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/365f6dc2/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 957ee68..e3aca06 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -250,7 +250,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
- private Observable<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
+ private List<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
if (logger.isDebugEnabled()) {
logger.debug("handleMessages with {} message", messages.size());
}
@@ -305,9 +305,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
});
//filter for success, send to the index(optional), ack
- return masterObservable
+ return (List<QueueMessage>) masterObservable
//take the max
- .buffer(indexProcessorFig.getBufferTime(), TimeUnit.MILLISECONDS, bufferSize)
+ .buffer(bufferSize)
//map them to index results and return them
.flatMap(indexEventResults -> {
IndexOperationMessage combined = new IndexOperationMessage();
@@ -330,7 +330,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
//return the queue messages to ack
.map(result -> result.getQueueMessage().get());
- });
+ })
+ .doOnError(t -> logger.error("Failed to process queuemessages",t))
+ .toBlocking().lastOrDefault(null);
}
@@ -520,7 +522,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
private void startWorker() {
synchronized (mutex) {
- Observable<QueueMessage> consumer =
+ Observable<List<QueueMessage>> consumer =
Observable.create(new Observable.OnSubscribe<List<QueueMessage>>() {
@Override
public void call(final Subscriber<? super List<QueueMessage>> subscriber) {
@@ -561,26 +563,28 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
})
//this won't block our read loop, just reads and proceeds
- .flatMap(messages ->
- {
- final int bufferSize = messages.size();
- return handleMessages(messages)
- .buffer(indexProcessorFig.getBufferTime(), TimeUnit.MILLISECONDS, bufferSize) //TODO how to ack multiple messages via buffer
- .doOnNext(messagesToAck -> {
- if (messagesToAck.size() == 0) {
- return;
- }
- try {
- //ack each message, but only if we didn't error.
- ack(messagesToAck);
- } catch (Exception e) {
- logger.error("failed to ack messages to sqs", messagesToAck.get(0).getMessageId(), e);
- //do not rethrow so we can process all of them
- }
- })
- .flatMap(messagesToAck -> Observable.from(messagesToAck));
+ .map(messages ->
+ {
+ if (messages == null || messages.size() == 0) {
+ return null;
}
- );
+
+ try {
+
+ List<QueueMessage> messagesToAck = handleMessages(messages);
+
+ if (messagesToAck == null || messagesToAck.size() == 0) {
+ return messagesToAck;
+ }
+ //ack each message, but only if we didn't error.
+ ack(messagesToAck);
+ return messagesToAck;
+ } catch (Exception e) {
+ logger.error("failed to ack messages to sqs", messages.get(0).getMessageId(), e);
+ return null;
+ //do not rethrow so we can process all of them
+ }
+ });
//start in the background
[09/50] [abbrv] usergrid git commit: refactor observable
Posted by sn...@apache.org.
refactor observable
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/17586ecb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/17586ecb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/17586ecb
Branch: refs/heads/asf-site
Commit: 17586ecb5dd38ee71b7edfd256fdecb50e615556
Parents: 890611d
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 6 08:26:46 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 6 08:26:46 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/17586ecb/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 37a9da5..e16de05 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -227,7 +227,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
- private Observable<IndexEventResult> handleMessages( final List<QueueMessage> messages ) {
+ private Observable<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
if (logger.isDebugEnabled()) {
logger.debug("handleMessages with {} message", messages.size());
}
@@ -290,7 +290,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
return indexProducer.put(combined)
- .flatMap(operationResult -> Observable.from(indexEventResults));
+ .flatMap(operationResult -> Observable.from(indexEventResults))
+ .map(result -> result.getQueueMessage());
});
@@ -448,7 +449,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
final Observable<IndexOperationMessage> merged = entityDeleteResults
.getEntitiesCompacted()
- .collect(() -> new ArrayList<>(),(list,item)-> list.add(item))
+ .collect(() -> new ArrayList<>(), (list, item) -> list.add(item))
.flatMap(collected -> entityDeleteResults.getIndexObservable()) ;
return merged;
}
@@ -499,7 +500,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
private void startWorker() {
synchronized (mutex) {
- Observable<IndexEventResult> consumer =
+ Observable<QueueMessage> consumer =
Observable.create(new Observable.OnSubscribe<List<QueueMessage>>() {
@Override
public void call(final Subscriber<? super List<QueueMessage>> subscriber) {
@@ -540,14 +541,13 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
})
//this won't block our read loop, just reads and proceeds
- .map(messages ->
+ .flatMap(messages ->
handleMessages(messages)
- .map(indexEventResult -> {
- ack(indexEventResult.getQueueMessage());
- return indexEventResult;
+ .doOnNext(message -> {
+ //ack each message, but only if we didn't error.
+ ack(message);
})
- .toBlocking().lastOrDefault(null)
- )//ack each message, but only if we didn't error. If we did, we'll want to log it and
+ )
.subscribeOn(Schedulers.newThread());
//start in the background
[03/50] [abbrv] usergrid git commit: fix subscribe for messages
Posted by sn...@apache.org.
fix subscribe for messages
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1a1d42e1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1a1d42e1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1a1d42e1
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 1a1d42e1f53cabf433442c17f614f9fcae418a22
Parents: b437f61
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 5 16:28:04 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 5 16:28:04 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1a1d42e1/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index e215d48..bf29c5a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -285,13 +285,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
- return indexProducer.put( combined )
- .flatMap( operationResult -> Observable.from( indexEventResults ) )
- //ack each message, but only if we didn't error. If we did, we'll want to log it and
- .map( indexEventResult -> {
- ack( indexEventResult.queueMessage );
- return indexEventResult;
- } );
+ return indexProducer.put(combined)
+ .flatMap(operationResult -> Observable.from(indexEventResults));
+
} );
}
@@ -538,7 +534,15 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
})
//this won't block our read loop, just reads and proceeds
- .flatMap( messages -> handleMessages( messages ) ).subscribeOn( Schedulers.newThread() );
+ .map(messages ->
+ handleMessages(messages)
+ .map(indexEventResult -> {
+ ack( indexEventResult.getQueueMessage() );
+ return indexEventResult;
+ })
+ .toBlocking().lastOrDefault(null)
+ )//ack each message, but only if we didn't error. If we did, we'll want to log it and
+ .subscribeOn( Schedulers.newThread() );
//start in the background
[11/50] [abbrv] usergrid git commit: move subscriber
Posted by sn...@apache.org.
move subscriber
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7dceb563
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7dceb563
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7dceb563
Branch: refs/heads/asf-site
Commit: 7dceb563eeb836a7dd09280b07af78c69ebb1793
Parents: 17586ec
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 6 08:39:21 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 6 08:39:21 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 5 ++-
.../core/rx/ExceptionBehaviorTest.java | 32 +++++++++++++++++++-
2 files changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7dceb563/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index e16de05..14d37b5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -547,12 +547,11 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack each message, but only if we didn't error.
ack(message);
})
- )
- .subscribeOn(Schedulers.newThread());
+ );
//start in the background
- final Subscription subscription = consumer.subscribe();
+ final Subscription subscription = consumer.subscribeOn(Schedulers.newThread()).subscribe();
subscriptions.add(subscription);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7dceb563/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
index be10d0a..8e4f4c4 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/ExceptionBehaviorTest.java
@@ -22,6 +22,7 @@ import org.junit.Test;
import rx.Observable;
import rx.Observer;
+import rx.schedulers.Schedulers;
/**
@@ -31,13 +32,21 @@ public class ExceptionBehaviorTest {
//this test shows toBlocking re-throws exceptions correctly
@Test( expected = TestException.class )
- public void throwOnBlocking() {
+ public void throwOnBlockingFirst() {
Observable.range( 0, 1 ).map( integer -> {
throw new TestException( "I throw and exception" );
} ).toBlocking().first();
}
+ @Test( expected = TestException.class )
+ public void throwOnBlockingLast() {
+
+ Observable.range( 0, 1 ).map( integer -> {
+ throw new TestException( "I throw and exception" );
+ } ).toBlocking().last();
+ }
+
//
// /**
// * This shows that no re-throw happens on subscribe. This is as designed, but not as expected
@@ -67,6 +76,27 @@ public class ExceptionBehaviorTest {
exceptionObserver.checkResult();
}
+ /**
+ * Tests working with observers
+ */
+ @Test( expected = TestException.class )
+ public void throwOnSubscribeObservableNewThread() throws Exception {
+
+ final ReThrowObserver exceptionObserver = new ReThrowObserver();
+
+ Observable.range( 0, 1 ).map(integer -> {
+ throw new TestException("I throw and exception");
+ })
+ .doOnError(t -> exceptionObserver.onError(t))
+ .subscribeOn(Schedulers.newThread())
+ .subscribe(exceptionObserver);
+
+ for(int i =0; i<5; i++) {
+ exceptionObserver.checkResult();
+ Thread.sleep(200);
+ }
+ }
+
private static final class TestException extends RuntimeException {
[15/50] [abbrv] usergrid git commit: cleanup observable,
add realistic timer, make sure messages that error are not ack'd
Posted by sn...@apache.org.
cleanup observable, add realistic timer, make sure messages that error are not ack'd
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9da0179f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9da0179f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9da0179f
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 9da0179f17d636d44ee8c4121c909632582aafe7
Parents: 7c5a864
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 6 18:29:56 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 6 18:29:56 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 67 ++++++++++----------
1 file changed, 33 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9da0179f/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 1bc70cd..0d7553e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -262,36 +262,44 @@ public class AmazonAsyncEventService implements AsyncEventService {
}catch (ClassCastException cce){
logger.error("Failed to deserialize message body",cce);
}
- logger.debug("Processing {} event", event);
if (event == null) {
logger.error("AsyncEvent type or event is null!");
- return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent()));
+ return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(),System.currentTimeMillis()));
}
+
+ final AsyncEvent thisEvent = event;
+ if(logger.isDebugEnabled()) {
+ logger.debug("Processing {} event", event);
+ }
+
try {
+ Observable<IndexOperationMessage> indexoperationObservable;
//merge each operation to a master observable;
if (event instanceof EdgeDeleteEvent) {
- return handleIndexOperation(message, queueMessage -> handleEdgeDelete(queueMessage));
+ indexoperationObservable = handleEdgeDelete(message);
} else if (event instanceof EdgeIndexEvent) {
- return handleIndexOperation(message, queueMessage -> handleEdgeIndex(queueMessage));
+ indexoperationObservable = handleEdgeIndex(message);
} else if (event instanceof EntityDeleteEvent) {
- return handleIndexOperation(message, queueMessage -> handleEntityDelete(queueMessage));
+ indexoperationObservable = handleEntityDelete(message);
} else if (event instanceof EntityIndexEvent) {
- return handleIndexOperation(message, queueMessage -> handleEntityIndexUpdate(queueMessage));
+ indexoperationObservable = handleEntityIndexUpdate(message);
} else if (event instanceof InitializeApplicationIndexEvent) {
//does not return observable
- handleInitializeApplicationIndex(message);
- return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent()));
+ handleInitializeApplicationIndex(event,message);
+ indexoperationObservable = Observable.just(new IndexOperationMessage());
} else {
- logger.error("Unknown EventType: {}", event);//TODO: print json instead
- return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent()));
+ throw new Exception("Unknown EventType");//TODO: print json instead
}
- } catch (Exception e) {
- logger.error("Failed to index entity", e, message);
- return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent()));
- } finally {
- messageCycle.update(System.currentTimeMillis() - event.getCreationTime());
+ //return type that can be indexed and ack'd later
+ return indexoperationObservable
+ .map(indexOperationMessage ->
+ new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage),thisEvent.getCreationTime())
+ );
+ } catch (Exception e) {
+ logger.error("Failed to index message: " + message.getMessageId(), e, message);
+ return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime()));
}
});
@@ -315,25 +323,13 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
return indexProducer.put(combined)
.flatMap(indexOperationMessage -> Observable.from(indexEventResults))
+ .doOnNext(indexEventResult -> messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()))
.map(result -> result.getQueueMessage().get());
});
}
- //transform index operation to
- private Observable<IndexEventResult> handleIndexOperation(QueueMessage queueMessage,
- Func1<QueueMessage, Observable<IndexOperationMessage>> operation
- ){
- try{
- return operation.call(queueMessage)
- .map(indexOperationMessage -> new IndexEventResult(Optional.fromNullable(queueMessage), Optional.fromNullable(indexOperationMessage)));
- }catch (Exception e){
- logger.error("failed to run index",e);
- return Observable.just( new IndexEventResult(Optional.fromNullable(queueMessage), Optional.<IndexOperationMessage>absent()));
- }
- }
-
@Override
public void queueInitializeApplicationIndex( final ApplicationScope applicationScope) {
@@ -479,11 +475,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
- public void handleInitializeApplicationIndex(final QueueMessage message) {
+ public void handleInitializeApplicationIndex(final AsyncEvent event, final QueueMessage message) {
Preconditions.checkNotNull(message, "Queue Message cannot be null for handleInitializeApplicationIndex");
-
- final AsyncEvent event = (AsyncEvent) message.getBody();
- Preconditions.checkNotNull( message, "QueueMessage Body cannot be null for handleInitializeApplicationIndex" );
Preconditions.checkArgument(event instanceof InitializeApplicationIndexEvent, String.format("Event Type for handleInitializeApplicationIndex must be APPLICATION_INDEX, got %s", event.getClass()));
final InitializeApplicationIndexEvent initializeApplicationIndexEvent =
@@ -492,7 +485,6 @@ public class AmazonAsyncEventService implements AsyncEventService {
final IndexLocationStrategy indexLocationStrategy = initializeApplicationIndexEvent.getIndexLocationStrategy();
final EntityIndex index = entityIndexFactory.createEntityIndex( indexLocationStrategy );
index.initialize();
- ack( message );
}
/**
@@ -607,15 +599,18 @@ public class AmazonAsyncEventService implements AsyncEventService {
public class IndexEventResult{
private final Optional<QueueMessage> queueMessage;
private final Optional<IndexOperationMessage> indexOperationMessage;
+ private final long creationTime;
- public IndexEventResult(Optional<QueueMessage> queueMessage, Optional<IndexOperationMessage> indexOperationMessage ){
+ public IndexEventResult(Optional<QueueMessage> queueMessage, Optional<IndexOperationMessage> indexOperationMessage, long creationTime){
this.queueMessage = queueMessage;
this.indexOperationMessage = indexOperationMessage;
+ this.creationTime = creationTime;
}
+
public Optional<QueueMessage> getQueueMessage() {
return queueMessage;
}
@@ -623,5 +618,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
public Optional<IndexOperationMessage> getIndexOperationMessage() {
return indexOperationMessage;
}
+
+ public long getCreationTime() {
+ return creationTime;
+ }
}
}
[17/50] [abbrv] usergrid git commit: remove integration tests
Posted by sn...@apache.org.
remove integration tests
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f005aceb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f005aceb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f005aceb
Branch: refs/heads/asf-site
Commit: f005aceb5cf872bed8eba796b81219aca520d0dc
Parents: 9da0179
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 7 11:54:24 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 7 11:54:24 2015 -0600
----------------------------------------------------------------------
stack/rest_integration_tests/README.md | 19 ---
stack/rest_integration_tests/config/default.js | 39 -----
stack/rest_integration_tests/config/index.js | 25 ---
stack/rest_integration_tests/index.js | 20 ---
stack/rest_integration_tests/lib/connections.js | 137 ---------------
stack/rest_integration_tests/lib/entities.js | 167 -------------------
.../rest_integration_tests/lib/notifications.js | 68 --------
stack/rest_integration_tests/lib/random.js | 74 --------
stack/rest_integration_tests/lib/response.js | 30 ----
stack/rest_integration_tests/lib/token.js | 49 ------
stack/rest_integration_tests/lib/urls.js | 37 ----
stack/rest_integration_tests/lib/users.js | 82 ---------
stack/rest_integration_tests/package.json | 18 --
.../test/authentication/management.js | 38 -----
.../test/authentication/org.js | 37 ----
.../test/authentication/resetPassword.js | 78 ---------
.../test/authentication/user.js | 40 -----
.../test/connections/create.js | 49 ------
.../test/connections/delete.js | 77 ---------
.../test/connections/get.js | 82 ---------
.../test/entities/create.js | 73 --------
.../test/entities/deleteAll.js | 37 ----
.../rest_integration_tests/test/entities/get.js | 51 ------
.../test/entities/update.js | 43 -----
stack/rest_integration_tests/test/main.js | 72 --------
stack/rest_integration_tests/test/mocha.opts | 3 -
.../test/notifications/create.js | 36 ----
.../test/queries/comparison.js | 58 -------
.../test/queries/contains.js | 116 -------------
.../test/queries/equals.js | 54 ------
.../test/queries/location.js | 42 -----
.../test/queries/order.js | 82 ---------
stack/rest_integration_tests/test/setup.js | 103 ------------
stack/rest_integration_tests/test/teardown.js | 65 --------
.../rest_integration_tests/test/users/create.js | 45 -----
35 files changed, 2046 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/README.md
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/README.md b/stack/rest_integration_tests/README.md
deleted file mode 100644
index c27ec39..0000000
--- a/stack/rest_integration_tests/README.md
+++ /dev/null
@@ -1,19 +0,0 @@
-#Usergrid RESTful Integration Tests
-
-These tests will run against a deployed instance of Usergrid and validate that APIs respond as expected. Tests require [Node.js](https://nodejs.org), [Mocha](http://mochajs.org), and [Should.js](http://shouldjs.github.io).
-
-Get Started:
-
-1. Install [Node.js](https://nodejs.org/download) version 0.12.1 or newer
-2. Install Mocha: `$ [sudo] npm -g install mocha`
-3. `$ cd` to `/integration_tests` and run `$ npm install`.
-4. Using `config/default.js` as a template, create a copy `config/override.js` and modify it according to your environment.
-5. Once configured, run `$ mocha test` from `/integration_tests` to perform tests.
-
-Notes:
-
-- Connections do not currently support org/app credentials. For tests to pass, you will need to give `Guest` POST rights to `/**` in the Usergrid authorizations table.
-- In order for notifications tests to pass, you will need to create an Apple notifier named `apple-dev` using a valid development APNS certificate.
-- In order to skip tests, you can append `.skip` to the test method, e.g.: `describe.skip()` or `it.skip()`.
-- Depending on your environment, certain tests may take longer than expected. You can override timeouts by setting `this.timeout(timeInMilliseconds)` and `this.slow(timeInMilliseconds)` inside the `describe()` method before the tests execute.
-- For more information on adding or modifying tests, check out the [Mocha](http://mochajs.org), and [Should.js](http://shouldjs.github.io) documentation.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/config/default.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/config/default.js b/stack/rest_integration_tests/config/default.js
deleted file mode 100644
index f3b8c17..0000000
--- a/stack/rest_integration_tests/config/default.js
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-module.exports = {
- serverUrl: "http://localhost:8080/",
- orgName: "test-organization", //must
- appName: "test-app", //must pre create app
- numberOfUsers: 5,
- numberOfEntitiesConsistent: 100,
- consistentChecks:3,
- numberOfEntities: 20,
- org: {
- clientId: "",
- clientSecret: ""
- },
- usersCollection: "users",
- entitiesTestCollection: "cats",
- genericTestCollection1: "dogs",
- genericTestCollection2: "horses",
- consumableTestCollection: "food",
- location: { // London
- latitude: 51.51279,
- longitude: -0.09184
- },
- notifierName: "noop-dev"
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/config/index.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/config/index.js b/stack/rest_integration_tests/config/index.js
deleted file mode 100644
index 98ac92d..0000000
--- a/stack/rest_integration_tests/config/index.js
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var _ = require("underscore");
-var config = require("./default.js")
-var override = require("./override.js")
-
-
-if (Object.keys(override).length > 0) {
- config = _.extend(config, override);
-}
-module.exports = config;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/index.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/index.js b/stack/rest_integration_tests/index.js
deleted file mode 100644
index 30a47b0..0000000
--- a/stack/rest_integration_tests/index.js
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require('./config/config.js');
-
-console.log('config is '+ JSON.stringify(config));
-
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/connections.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/connections.js b/stack/rest_integration_tests/lib/connections.js
deleted file mode 100644
index 841281d..0000000
--- a/stack/rest_integration_tests/lib/connections.js
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-var urls = require("./urls");
-var random = require("./random");
-var responseLib = require("./response");
-var async = require('async');
-var request = require("request");
-var sleep = require('sleep');
-module.exports = {
- create: function(fromCollection, toCollection, relationship, cb) {
- async.parallel({
- from: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + fromCollection),
- json: true
- }, function(e, r, body) {
- cb(e, body.entities[0]);
- });
- },
- to: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + toCollection),
- json: true
- }, function(e, r, body) {
- cb(e, body.entities[0]);
- });
- }
- }, function(err, results) {
- var url = urls.appUrl() +
- fromCollection + "/" +
- results.from.uuid + "/" +
- relationship + "/" +
- results.to.uuid;
- url = urls.appendOrgCredentials(url)
- request.post({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- });
- });
- },
- get: function(fromCollection, toCollection, relationship, cb) {
- async.parallel({
- from: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + fromCollection + "?limit=1"),
- json: true
- }, function(e, r, body) {
- var o = {
- parent: body.entities[0]
- }
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + fromCollection + "/" + o.parent.uuid + "/" + relationship),
- json: true
- }, function(e, r, body) {
- o.related = body.entities;
- cb(e, o);
- });
- });
- },
- to: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + toCollection + "?limit=1"),
- json: true
- }, function(e, r, body) {
- var o = {
- parent: body.entities[0]
- }
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + toCollection + "/" + o.parent.uuid + "/connecting/" + relationship),
- json: true
- }, function(e, r, body) {
- o.related = body.entities;
- cb(e, o);
- });
- });
- }
- }, function(err, results) {
- cb(err, results);
- });
- },
- delete: function(fromCollection, toCollection, relationship, cb) {
- async.parallel({
- from: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + fromCollection),
- json: true
- }, function(e, r, body) {
- cb(e, body.entities[0]);
- });
- },
- to: function(cb) {
- request.get({
- url: urls.appendOrgCredentials(urls.appUrl() + toCollection),
- json: true
- }, function(e, r, body) {
- cb(e, body.entities[0]);
- });
- }
- }, function(err, results) {
- var url = urls.appUrl() +
- fromCollection + "/" +
- results.from.uuid + "/" +
- relationship + "/" +
- //toCollection + "/" +
- results.to.uuid;
- url = urls.appendOrgCredentials(url);
- sleep.sleep(1);
- request.del({
- url: url,
- json: true
- }, function(e, r, body) {
- sleep.sleep(1);
- module.exports.get(fromCollection, toCollection, relationship, function(err, results) {
- cb(err, results);
- });
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/entities.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/entities.js b/stack/rest_integration_tests/lib/entities.js
deleted file mode 100644
index fcdcfb9..0000000
--- a/stack/rest_integration_tests/lib/entities.js
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-var urls = require("./urls");
-var random = require("./random");
-var responseLib = require("./response");
-var async = require('async');
-var request = require("request");
-
-module.exports = {
- create: function(collection, numberOfEntities, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection);
- var requestArray = []
- geos = random.geo(config.location, 2000, numberOfEntities);
- // console.log(geos);
- for (var i = 0; i < numberOfEntities; i++) {
- requestArray.push({
- consistentProperty: "somethingConsistent",
- randomProperty: "somethingRandom - " + random.randomString(10),
- intProperty: random.randomNumber(5),
- optionsProperty: random.abc(),
- location: geos[i],
- title: "A Tale of Two Cities"
- });
- }
- request.post({
- url: url,
- json: true,
- body: requestArray
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- });
- },
- createEach: function(collection, numberOfEntities, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection);
- var requestArray = []
- geos = random.geo(config.location, 2000, numberOfEntities);
- // console.log(geos);
- for (var i = 0; i < numberOfEntities; i++) {
- requestArray.push({
- consistentProperty: "somethingConsistent",
- randomProperty: "somethingRandom - " + random.randomString(10),
- intProperty: random.randomNumber(5),
- optionsProperty: random.abc(),
- location: geos[i],
- title: "A Tale of Two Cities"
- });
- }
- var returnBody = [];
- async.each(requestArray, function(options, cb) {
- request.post({
- url: url,
- json: true,
- body: options
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- var entity = body && body.entities ? body.entities.pop() : null;
- entity && returnBody.push(entity);
- cb(error, error ? error : entity);
- });
- }, function(err,bodies) {
- cb(err,returnBody);
- });
-
- },
- deleteAll: function(collection, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection);
- deleteAllEntities(collection, function(e) {
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- })
- },
- update: function(collection, uuid, body, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection + "/" + uuid);
- request.put({
- url: url,
- body: body,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- },
- get: function(collection, numberOfEntities, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection + "?limit=" + numberOfEntities.toString());
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- },
- getByUuid: function(collection, uuid, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection + "/"+uuid);
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- },
- getWithQuery: function(collection, query, numberOfEntities, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection + "?ql=" + encodeURIComponent(query) + "&limit=" + numberOfEntities.toString());
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- })
- }
-};
-
-function deleteAllEntities(collection, cb) {
- var url = urls.appendOrgCredentials(urls.appUrl() + collection);
- request.get({
- url: url,
- json: true
- }, function(e, r, body) {
- if (body.count === undefined) {
- cb("The 'count' property is not defined at " + url);
- } else if (body.count > 0) {
- var deletes = [];
- for (var i = 0; i < body.count; i++) {
- deletes.push({
- url: urls.appendOrgCredentials(urls.appUrl() + collection + "/" + body.entities[i].uuid),
- json: true
- });
- }
- async.each(deletes, function(options, cb) {
- request.del(options, function(e) {
- cb(e);
- });
- }, function(err) {
- setTimeout(function() {
- deleteAllEntities(collection, function(e) {
- cb(e);
- });
- }, 600); // Mandatory, since it seems to not retrieve entities if you make a request in < 600ms
- });
- } else {
- cb();
- }
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/notifications.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/notifications.js b/stack/rest_integration_tests/lib/notifications.js
deleted file mode 100644
index 7834e2c..0000000
--- a/stack/rest_integration_tests/lib/notifications.js
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-var urls = require("./urls");
-var random = require("./random");
-var responseLib = require("./response");
-var async = require('async');
-var request = require("request");
-var random = require("./random");
-
-
-module.exports = {
-
- create: function(message, cb) {
- var notifierName = config.notifierName + "_" + random.randomString(5);
-
- // Need to ensure at least one device exists in the devices collection
- request.post({
- url: urls.appendOrgCredentials(urls.appUrl() + "notifiers"),
- json: true,
- body: {
- name: notifierName,
- provider: "noop"
- }
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- if(error){
- return cb(error)
- }
- request.post({
- url: urls.appendOrgCredentials(urls.appUrl() + "devices"),
- json: true,
- body: {
- name: "testDevice"
- }
- }, function(e, r, body) {
- payload = {};
- payload[notifierName] = message;
- request.post({
- url: urls.appendOrgCredentials(urls.appUrl() + "devices;ql=/notifications"),
- json: true,
- body: {
- payloads: payload
- }
- }, function(e, r, body) {
- var error = responseLib.getError(e, r);
- cb(error, error ? error : body);
- });
- });
- });
-
-
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/random.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/random.js b/stack/rest_integration_tests/lib/random.js
deleted file mode 100644
index d5dce65..0000000
--- a/stack/rest_integration_tests/lib/random.js
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-module.exports = {};
-module.exports.randomString = function randomString(length) {
- var text = "";
- var possible = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
-
- for (var i = 0; i < length; i++) {
- text += possible.charAt(Math.floor(Math.random() * possible.length));
- }
- return text;
-}
-
-module.exports.abc = function abc() {
- letters = ["aaa bbb ccc", "ddd eee fff", "ggg hhh iii", "jjj kkk lll"];
- return letters[Math.floor(Math.random() * letters.length)];
-}
-
-module.exports.randomNumber = function randomNumber(length) {
- var text = "";
- var possible = "0123456789";
-
- for (var i = 0; i < length; i++) {
- text += possible.charAt(Math.floor(Math.random() * possible.length));
- }
- return parseInt(text);
-}
-
-module.exports.randomEntity = function randomEntity(entitiesArray) {
- return entitiesArray[Math.floor(Math.random()*entitiesArray.length)];
-}
-
-module.exports.geo = function geo(center, radius, count) {
- var points = [];
- for (var i = 0; i < count; i++) {
- points.push(randomGeo(center, radius));
- }
- return points;
-}
-
-function randomGeo(center, radius) {
- var y0 = center.latitude;
- var x0 = center.longitude;
- var rd = radius / 111300;
-
- var u = Math.random();
- var v = Math.random();
-
- var w = rd * Math.sqrt(u);
- var t = 2 * Math.PI * v;
- var x = w * Math.cos(t);
- var y = w * Math.sin(t);
-
- // var xp = x / Math.cos(y0);
-
- return {
- 'latitude': y + y0,
- 'longitude': x + x0
- };
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/response.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/response.js b/stack/rest_integration_tests/lib/response.js
deleted file mode 100644
index 7aab031..0000000
--- a/stack/rest_integration_tests/lib/response.js
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-module.exports = {};
-module.exports.getError = function(err, response) {
- return err || (response.statusCode >= 400 ? response.body : null)
-};
-
-module.exports.distanceInMeters = function(location1, location2) {
- var R = 6371000;
- var a = 0.5 - Math.cos((location2.latitude - location1.latitude) * Math.PI / 180) / 2 +
- Math.cos(location1.latitude * Math.PI / 180) * Math.cos(location2.latitude * Math.PI / 180) *
- (1 - Math.cos((location2.longitude - location1.longitude) * Math.PI / 180)) / 2;
-
- var distance = R * 2 * Math.asin(Math.sqrt(a));
- return distance;
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/token.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/token.js b/stack/rest_integration_tests/lib/token.js
deleted file mode 100644
index 96a3522..0000000
--- a/stack/rest_integration_tests/lib/token.js
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-var urls = require("./urls");
-var responseLib = require("./response");
-var request = require("request");
-
-module.exports= {
- getOrgToken: function (cb) {
- var managementUrl = urls.managementUrl();
-
- var options = {
- uri: managementUrl + "token",
- method: 'POST',
- json: {client_id: config.org.clientId, client_secret: config.org.clientSecret, grant_type: "client_credentials"}
- };
- request(options, function (err, response, body) {
- var error = responseLib.getError(err,response);
- cb(error, body);
- });
- },
- getManagementToken: function (username, password, cb) {
- var managementUrl = urls.managementUrl();
- var options = {
- uri: managementUrl + "token",
- method: 'POST',
- json: {username: username, password: password, grant_type: "password"}
- };
- request.post(options, function (err, response, body) {
- var error = responseLib.getError(err,response);
- cb(error,body);
- });
- }
-
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/urls.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/urls.js b/stack/rest_integration_tests/lib/urls.js
deleted file mode 100644
index 2edfd57..0000000
--- a/stack/rest_integration_tests/lib/urls.js
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var config = require("../config");
-module.exports = {
- appUrl: function () {
- return config.serverUrl + config.orgName + "/" + config.appName + "/";
- },
- managementUrl: function () {
- return config.serverUrl + "management/";
- },
- appendAccessToken: function(url,tokenData){
- if(tokenData == null){
- return url;
- }
- var token = tokenData.access_token || tokenData;
- return url + (url.indexOf("?") >= 0 ? "&" : "?" ) + "access_token="+token;
- },
- appendOrgCredentials: function(url, clientId, clientSecret){
- clientId = clientId || config.org.clientId;
- clientSecret = clientSecret || config.org.clientSecret;
- return url + (url.indexOf("?") >= 0 ? "&" : "?" ) + "client_id="+clientId+"&client_secret="+clientSecret;
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/lib/users.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/lib/users.js b/stack/rest_integration_tests/lib/users.js
deleted file mode 100644
index 0a47324..0000000
--- a/stack/rest_integration_tests/lib/users.js
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var request = require("request");
-var token = require("./token");
-var urls = require("./urls");
-var responseLib = require("./response");
-module.exports = {};
-
-module.exports.add = function(user, cb) {
- request.post(urls.appendOrgCredentials(urls.appUrl() + "users"), {
- json: user
- }, function(err, response, body) {
- var error = responseLib.getError(err, response);
- cb(error, error ? null : body.entities.pop());
- });
-};
-
-module.exports.login = function(username, password, cb) {
- request.post({
- url: urls.appUrl() + "token",
- json: {
- username: username,
- password: password,
- grant_type: "password"
- }
- }, function(err, response, body) {
- var error = responseLib.getError(err, response);
- cb(error, body);
- });
-};
-
-module.exports.resetPassword = function(username, oldpassword, newpassword, cb) {
- request.post({
- uri: urls.appUrl() + "users/" + username + "/password",
- json: {
- oldpassword: oldpassword,
- newpassword: newpassword
- }
- }, function(e, r, body) {
- cb(e, r, body);
- });
-};
-
-module.exports.resetPasswordAsAdmin = function(username, newpassword, cb) {
- request.post({
- uri: urls.appendOrgCredentials(urls.appUrl() + "users/" + username + "/password"),
- json: {
- newpassword: newpassword
- }
- }, function(e, r, body) {
- cb(e, r, body);
- });
-};
-
-module.exports.addToRole = function(username, role, cb) {
- request.post(urls.appendOrgCredentials(urls.appUrl() + "roles/" + role + "/users/" + username), null, function(err, response, body) {
- var error = responseLib.getError(err, response);
- cb(error);
- });
-};
-
-module.exports.get = function(username, cb) {
- request.get(urls.appendOrgCredentials(urls.appUrl() + "users/" + username), function(err, response, body) {
- var json = JSON.parse(body);
- var error = response.statusCode === 404 ? null : responseLib.getError(err, response);
- cb(error, error ? null : response.statusCode === 404 ? null : json.entities.pop());
- })
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/package.json
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/package.json b/stack/rest_integration_tests/package.json
deleted file mode 100644
index bad9cbc..0000000
--- a/stack/rest_integration_tests/package.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "name": "Usergrid_Tests",
- "version": "0.0.1",
- "devDependencies": {
- "async": "^1.2.1",
- "colors": "~1.1.2",
- "i": "~0.3.3",
- "mocha": "~2.2.5",
- "request": "~2.58.0",
- "should": "~6.0.3",
- "sleep": "^3.0.0",
- "underscore": "^1.8.3",
- "uuid": "^2.0.1"
- },
- "engines": {
- "node": ">0.12.1"
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/authentication/management.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/authentication/management.js b/stack/rest_integration_tests/test/authentication/management.js
deleted file mode 100644
index bef2a96..0000000
--- a/stack/rest_integration_tests/test/authentication/management.js
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var users = require("../../lib/users")
-var config = require("../../config")
-var should = require("should")
-var setup = require("../setup")
-module.exports = {};
-
-module.exports.test = function() {
- describe('get a management token', function() {
- it('should return valid token', function(done) {
- var admin = setup.admins[0];
- users.login(admin.username, admin.password, function(err, body) {
- should(err).be.null;
- body.should.have.property('access_token').and.have.lengthOf(63);;
- body.should.have.property('expires_in');
- body.should.have.property('expires_in').which.is.a.Number;
- body.user.username.should.equal(admin.username);
- done();
- });
- });
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/authentication/org.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/authentication/org.js b/stack/rest_integration_tests/test/authentication/org.js
deleted file mode 100644
index 95d182a..0000000
--- a/stack/rest_integration_tests/test/authentication/org.js
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var token = require("../../lib/token")
-var config = require("../../config")
-var should = require("should")
-var setup = require("../setup")
-module.exports = {};
-
-module.exports.test = function() {
- describe('get an org token', function() {
- it('should return valid token', function(done) {
- token.getOrgToken(function(err, tokenData) {
- should(err).be.null;
- tokenData.should.have.property('access_token').and.have.lengthOf(63);;
- tokenData.should.have.property('expires_in');
- tokenData.should.have.property('expires_in').which.is.a.Number;
- tokenData.organization.name.should.equal(config.orgName)
- done();
- });
- });
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/authentication/resetPassword.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/authentication/resetPassword.js b/stack/rest_integration_tests/test/authentication/resetPassword.js
deleted file mode 100644
index 86d01a6..0000000
--- a/stack/rest_integration_tests/test/authentication/resetPassword.js
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var token = require("../../lib/token")
-var users = require("../../lib/users")
-var config = require("../../config")
-var should = require("should")
-var setup = require("../setup")
-var async = require("async");
-
-module.exports = {};
-
-module.exports.test = function() {
- describe('reset a user password', function() {
- it('should reset a user\'s password only when the correct old password is provided', function(done) {
- var user = setup.users[0];
- users.login(user.username, user.password, function(err, body) {
- should(err).be.null;
- body.should.have.property('access_token').and.have.lengthOf(63);
- async.parallel({
- bad: function(cb) {
- users.resetPassword(user.username, user.password + "_bad", user.password + "_badnew", function(e, r, body) {
- cb(e, {
- r: r,
- body: body
- });
- });
- },
- good: function(cb) {
- users.resetPassword(user.username, user.password, user.password + "_goodnew", function(e, r, body) {
- cb(e, {
- r: r,
- body: body
- });
- });
- }
- },
- function(err, results) {
- results.bad.r.statusCode.should.equal(400);
- results.bad.body.should.have.property('error').which.equal('auth_invalid_username_or_password');
- results.bad.body.should.have.property('exception').which.equal('org.apache.usergrid.management.exceptions.IncorrectPasswordException');
- results.bad.body.should.have.property('error_description').which.equal('Unable to authenticate due to username or password being incorrect');
-
- results.good.r.statusCode.should.equal(200);
- results.good.body.should.have.property('action').which.equal('set user password');
- results.good.body.should.have.property('duration');
-
- done();
- });
-
- });
- });
- it('should reset a user\'s password using org credentials', function(done) {
- var user = setup.users[0];
- users.resetPasswordAsAdmin(user.username, user.password, function(e, r, body) {
- r.statusCode.should.equal(200);
- body.should.have.property('action').which.equal('set user password');
- body.should.have.property('duration');
-
- done();
- });
- })
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/authentication/user.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/authentication/user.js b/stack/rest_integration_tests/test/authentication/user.js
deleted file mode 100644
index 8faddd3..0000000
--- a/stack/rest_integration_tests/test/authentication/user.js
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var token = require("../../lib/token")
-var users = require("../../lib/users")
-var config = require("../../config")
-var should = require("should")
-var setup = require("../setup")
-
-module.exports = {};
-
-module.exports.test = function() {
- describe('get a user token', function() {
- it('should return valid token', function(done) {
- var user = setup.users[0];
- users.login(user.username, user.password, function(err, body) {
- should(err).be.null;
- body.should.have.property('access_token').and.have.lengthOf(63);
- body.should.have.property('expires_in');
- body.should.have.property('expires_in').which.is.a.Number;
- body.user.username.should.equal(user.username);
- done();
- });
- });
- });
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/connections/create.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/connections/create.js b/stack/rest_integration_tests/test/connections/create.js
deleted file mode 100644
index ca9ce84..0000000
--- a/stack/rest_integration_tests/test/connections/create.js
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var connections = require("../../lib/connections");
-var should = require("should");
-var config = require('../../config');
-var inflect = require('i')();
-
-module.exports = {
- test: function() {
- describe("create connection", function() {
- it("should connect " + config.genericTestCollection1 + "[0] to " + config.consumableTestCollection + "[0] via the relationship 'consumed'",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.create(config.genericTestCollection1, config.consumableTestCollection, "consumed", function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.entities[0].type.should.equal(config.consumableTestCollection);
- done();
- })
- });
- it("should connect " + config.genericTestCollection1 + "[0] to " + config.genericTestCollection2 + "[0] via the relationship 'likes'",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.create(config.genericTestCollection1, config.genericTestCollection2, "likes", function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.entities[0].type.should.equal(inflect.singularize(config.genericTestCollection2));
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/connections/delete.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/connections/delete.js b/stack/rest_integration_tests/test/connections/delete.js
deleted file mode 100644
index 9014a02..0000000
--- a/stack/rest_integration_tests/test/connections/delete.js
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var connections = require("../../lib/connections");
-var should = require("should");
-var config = require('../../config');
-var util = require('util');
-var inflect = require('i')();
-
-module.exports = {
- test: function() {
- describe("delete connections", function() {
- var rel1 = "consumed";
- it("should delete the '" + rel1 + "' connection between " + config.genericTestCollection1 + "[0] and " + config.consumableTestCollection + "[0]",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.delete(config.genericTestCollection1, config.consumableTestCollection, rel1, function(err, r) {
- should(err).be.null;
- if (r.from.parent.metadata.hasOwnProperty("connections")) {
- r.from.parent.metadata.connections.should.not.have.property(rel1);
- } else {
- r.from.parent.metadata.should.not.have.property("connections");
- }
- r.from.parent.metadata.should.not.have.property("connecting");
- r.from.related.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- if (r.to.parent.metadata.hasOwnProperty("connecting")) {
- r.to.parent.metadata.connecting.should.not.have.property(rel1);
- } else {
- r.to.parent.metadata.should.not.have.property("connecting");
- }
- r.to.related.should.be.an.instanceOf(Array).and.have.lengthOf(0);
-
- done();
- })
- });
- var rel2 = "likes";
- it("should delete the '" + rel2 + "' connection between " + config.genericTestCollection1 + "[0] and " + config.genericTestCollection2 + "[0]",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.delete(config.genericTestCollection1, config.genericTestCollection2, rel2, function(err, r) {
- should(err).be.null;
- if (r.from.parent.metadata.hasOwnProperty("connections")) {
- r.from.parent.metadata.connections.should.not.have.property(rel2);
- } else {
- r.from.parent.metadata.should.not.have.property("connections");
- }
- r.from.parent.metadata.should.not.have.property("connecting");
- r.from.related.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- if (r.to.parent.metadata.hasOwnProperty("connecting")) {
- r.to.parent.metadata.connecting.should.not.have.property(rel2);
- } else {
- r.to.parent.metadata.should.not.have.property("connecting");
- }
- r.from.parent.metadata.should.not.have.property("connections");
- r.to.related.should.be.an.instanceOf(Array).and.have.lengthOf(0);
-
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/connections/get.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/connections/get.js b/stack/rest_integration_tests/test/connections/get.js
deleted file mode 100644
index cea7bbe..0000000
--- a/stack/rest_integration_tests/test/connections/get.js
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var connections = require("../../lib/connections");
-var should = require("should");
-var config = require('../../config');
-var util = require('util');
-var inflect = require('i')();
-
-module.exports = {
- test: function() {
- describe("get connections", function() {
- var rel1 = "consumed";
- it("should see " + config.genericTestCollection1 + "[0] connected to " + config.consumableTestCollection + "[0] by the relationship '" + rel1 + "'",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.get(config.genericTestCollection1, config.consumableTestCollection, rel1, function(err, r) {
- should(err).be.null;
-
- r.from.parent.metadata.connections.should.have.property(rel1)
- r.from.parent.metadata.connections[rel1].should.equal(
- util.format("/%s/%s/%s", inflect.pluralize(config.genericTestCollection1), r.from.parent.uuid, rel1)
- );
- r.from.parent.type.should.equal(inflect.singularize(config.genericTestCollection1));
- r.from.related[0].uuid.should.equal(r.to.parent.uuid);
- r.from.related[0].type.should.equal(inflect.singularize(config.consumableTestCollection));
-
- r.to.parent.metadata.connecting.should.have.property(rel1)
- r.to.parent.metadata.connecting[rel1].should.equal(
- util.format("/%s/%s/connecting/%s", inflect.pluralize(config.consumableTestCollection), r.to.parent.uuid, rel1)
- );
- r.to.parent.type.should.equal(inflect.singularize(config.consumableTestCollection));
- r.to.related[0].uuid.should.equal(r.from.parent.uuid);
- r.to.related[0].type.should.equal(inflect.singularize(config.genericTestCollection1));
-
- done();
- })
- });
- var rel2 = "likes";
- it("should see " + config.genericTestCollection1 + "[0] connected to " + config.genericTestCollection2 + "[0] by the relationship '" + rel2 + "'",
- function(done) {
- this.slow(10000);
- this.timeout(15000);
- connections.get(config.genericTestCollection1, config.genericTestCollection2, rel2, function(err, r) {
- should(err).be.null;
-
- r.from.parent.metadata.connections.should.have.property(rel2)
- r.from.parent.metadata.connections[rel2].should.equal(
- util.format("/%s/%s/%s", inflect.pluralize(config.genericTestCollection1), r.from.parent.uuid, rel2)
- );
- r.from.parent.type.should.equal(inflect.singularize(config.genericTestCollection1));
- r.from.related[0].uuid.should.equal(r.to.parent.uuid);
- r.from.related[0].type.should.equal(inflect.singularize(config.genericTestCollection2));
-
- r.to.parent.metadata.connecting.should.have.property(rel2)
- r.to.parent.metadata.connecting[rel2].should.equal(
- util.format("/%s/%s/connecting/%s", inflect.pluralize(config.genericTestCollection2), r.to.parent.uuid, rel2)
- );
- r.to.parent.type.should.equal(inflect.singularize(config.genericTestCollection2));
- r.to.related[0].uuid.should.equal(r.from.parent.uuid);
- r.to.related[0].type.should.equal(inflect.singularize(config.genericTestCollection1));
-
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/entities/create.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/entities/create.js b/stack/rest_integration_tests/test/entities/create.js
deleted file mode 100644
index faccee8..0000000
--- a/stack/rest_integration_tests/test/entities/create.js
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var config = require('../../config');
-var async = require('async');
-module.exports = {
- test: function() {
- var uuid = require("uuid");
- var collectionName = "resttest_"+ uuid.v1().toString().replace("-", "");
-
- describe("create entities", function() {
- var numberOfRecords = config.numberOfEntities;
-
- it("should create " + numberOfRecords.toString() + " entities in the " + config.entitiesTestCollection + " collection", function(done) {
- this.slow(numberOfRecords * 500);
- entities.create(config.entitiesTestCollection, numberOfRecords, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(numberOfRecords);
- body.entities.forEach(function(entity) {
- entity.should.have.property("uuid").and.match(/(\w{8}(-\w{4}){3}-\w{12}?)/);
- });
- done();
- })
- });
- var numberOfRecordsConsistent = config.numberOfEntitiesConsistent;
- it("should create " + numberOfRecordsConsistent.toString() + " entities in the " + collectionName + " collection and check for consistency", function(done) {
- this.timeout(60000*100);
- bodyMap = {};
-
- //this.slow(numberOfRecordsConsistent * 500);
- async.times(config.consistentChecks, function(n, next) {
- entities.createEach(collectionName+n, numberOfRecordsConsistent, function (err, bodies) {
- should(err).be.null;
- bodies.should.be.an.instanceOf(Array).and.have.lengthOf(numberOfRecordsConsistent);
- bodies.forEach(function (body) {
- bodyMap[body.uuid] = body;
- });
- entities.get(collectionName+n, numberOfRecordsConsistent, function (err, entityArray) {
- should(err).be.null;
- next(err,entityArray);
- });
- });
- },
- function(err,list){
- list.forEach(function (listOfEntities) {
- listOfEntities.entities.forEach(function(entity){
- delete(bodyMap[entity.uuid]);
- });
- });
- Object.keys(bodyMap).length && console.log(JSON.stringify(bodyMap));
- should(Object.keys(bodyMap)).have.lengthOf(0);
- done();
- }
- );
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/entities/deleteAll.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/entities/deleteAll.js b/stack/rest_integration_tests/test/entities/deleteAll.js
deleted file mode 100644
index daa20ea..0000000
--- a/stack/rest_integration_tests/test/entities/deleteAll.js
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var config = require('../../config');
-
-module.exports = {
- test: function(collectionName) {
- collectionName = collectionName ? collectionName : config.entitiesTestCollection;
- describe("delete entities", function() {
- it("should delete all entities from the " + collectionName + " collection", function(done) {
- this.timeout(60000);
- this.slow(30000);
- entities.deleteAll(collectionName, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(0);
- body.count.should.equal(0);
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/entities/get.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/entities/get.js b/stack/rest_integration_tests/test/entities/get.js
deleted file mode 100644
index 665b1e6..0000000
--- a/stack/rest_integration_tests/test/entities/get.js
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require("async");
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("get entities", function() {
- it("should get 1 entity", function(done) {
- entities.get(config.entitiesTestCollection, 1, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.count.should.equal(1);
- done();
- })
- });
- it("should get 4 entities", function(done) {
- entities.get(config.entitiesTestCollection, 4, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(4);
- body.count.should.equal(4);
- done();
- })
- });
- it("should get 18 entities", function(done) {
- entities.get(config.entitiesTestCollection, 18, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(18);
- body.count.should.equal(18);
- done();
- })
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/entities/update.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/entities/update.js b/stack/rest_integration_tests/test/entities/update.js
deleted file mode 100644
index 04f4a97..0000000
--- a/stack/rest_integration_tests/test/entities/update.js
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var random = require("../../lib/random");
-var should = require("should");
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("update entity", function() {
- it("should get a random entity and set 'newProperty' to 'BANJO'", function(done) {
- this.timeout(10000);
- this.slow(5000);
- entities.get(config.entitiesTestCollection, random.randomNumber(10), function(err, body) {
- var payload = {
- newProperty: "BANJO"
- }
- should(body.entities[0].newProperty).not.exist;
- entities.update(config.entitiesTestCollection, body.entities[body.entities.length - 1].uuid, payload, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.entities[0].newProperty.should.equal("BANJO");
- done();
- });
- });
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/main.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/main.js b/stack/rest_integration_tests/test/main.js
deleted file mode 100644
index 24b6cfe..0000000
--- a/stack/rest_integration_tests/test/main.js
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var should = require("should");
-var config = require("../config/");
-var setup = require("./setup");
-var teardown = require("./teardown");
-var async = require('async');
-var request = require('request');
-var colors = require('colors');
-
-var entities = require('../lib/entities');
-
-describe("baas 2.0 tests", function() {
- before(function(done) {
- setup.do(function() {
- done();
- })
- });
- describe("authentication", function() {
- require("./authentication/user.js").test();
- require("./authentication/resetPassword.js").test();
- require("./authentication/management.js").test();
- require("./authentication/org.js").test();
- });
- describe("users", function() {
- require("./users/create.js").test();
- });
- describe("entities", function() {
- require("./entities/create.js").test();
- require("./entities/get.js").test();
- require("./entities/update.js").test();
- require("./entities/deleteAll.js").test();
- });
- describe("connections", function() {
- require('./connections/create.js').test();
- require('./connections/get.js').test();
- require('./connections/delete.js').test();
- });
- describe("queries", function() {
- require('./queries/equals.js').test();
- require('./queries/contains.js').test();
- require('./queries/order.js').test();
- require('./queries/comparison.js').test();
- require('./queries/location.js').test();
- });
- describe("notifications", function() {
- // Requires an apple notifier to be created in BaaS portal prior to running this test.
- // See: http://apigee.com/docs/app-services/content/creating-notifiers
- require('./notifications/create.js').test();
- });
-
- after(function(done) {
- this.timeout(40000);
- teardown.do(function() {
- done();
- });
- });
-});
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/mocha.opts
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/mocha.opts b/stack/rest_integration_tests/test/mocha.opts
deleted file mode 100644
index 9e2480f..0000000
--- a/stack/rest_integration_tests/test/mocha.opts
+++ /dev/null
@@ -1,3 +0,0 @@
---ui bdd
---recursive
---timeout 5000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/notifications/create.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/notifications/create.js b/stack/rest_integration_tests/test/notifications/create.js
deleted file mode 100644
index b3a2bb7..0000000
--- a/stack/rest_integration_tests/test/notifications/create.js
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var should = require("should");
-var uuid = require("uuid");
-var notifications = require("../../lib/notifications");
-
-module.exports = {
- test: function() {
- // Requires an apple notifier to be created in BaaS portal prior to running this test.
- // See: http://apigee.com/docs/app-services/content/creating-notifiers
- describe("create a notification", function() {
- it("should successfully create a notification", function(done) {
- notifications.create("Hello World!", function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array).and.have.lengthOf(1);
- body.entities[0].state.should.equal('FINISHED');
- done();
- });
- });
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/comparison.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/comparison.js b/stack/rest_integration_tests/test/queries/comparison.js
deleted file mode 100644
index 5fedbdb..0000000
--- a/stack/rest_integration_tests/test/queries/comparison.js
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("filter " + config.genericTestCollection2 + " with '>' and '<' queries", function() {
- var query = "where intProperty > 30000";
- numberOfEntities = Math.min(config.numberOfEntities, 10);
- it('should return a subset of results ' + query, function(done) {
- this.timeout(10000);
- entities.getWithQuery(config.genericTestCollection2, query, numberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.intProperty.should.be.greaterThan(30000);
- });
- }
- done();
- });
- });
- var query = "where intProperty > 30000 && intProperty < 40000";
- it('should return a subset of results ' + query, function(done) {
- this.timeout(10000);
- entities.getWithQuery(config.genericTestCollection2, query, numberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.intProperty.should.be.greaterThan(30000).and.lessThan(40000);
- });
- }
- done();
- });
- });
- });
- }
-};
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f005aceb/stack/rest_integration_tests/test/queries/contains.js
----------------------------------------------------------------------
diff --git a/stack/rest_integration_tests/test/queries/contains.js b/stack/rest_integration_tests/test/queries/contains.js
deleted file mode 100644
index 01c8ef1..0000000
--- a/stack/rest_integration_tests/test/queries/contains.js
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-'use strict';
-var entities = require("../../lib/entities");
-var should = require("should");
-var async = require('async');
-var config = require('../../config');
-
-module.exports = {
- test: function() {
- describe("filter " + config.genericTestCollection1 + " with 'contains' queries", function(done) {
- var query1 = "where consistentProperty contains 'somethingConsistent'";
- var maxNumberOfEntities = Math.max(config.numberOfEntities, 100);
- it('should return ' + config.numberOfEntities + ' results ' + query1, function(done) {
- this.timeout(30000);
- entities.getWithQuery(config.genericTestCollection1, query1, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.equal(config.numberOfEntities);
- body.entities.forEach(function(entity) {
- entity.consistentProperty.should.equal('somethingConsistent');
- });
- done();
- });
- });
- var query2 = "where consistentProperty contains '*ethi*'";
- // skipping this test for now since it doesn't work in 1.0
- it.skip('should return ' + config.numberOfEntities + ' results ' + query2, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query2, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.equal(config.numberOfEntities);
- body.entities.forEach(function(entity) {
- entity.consistentProperty.should.equal('somethingConsistent');
- });
- done();
- });
- });
- var query3 = "where optionsProperty contains 'aaa*'";
- // this should be updated when running tests against 2.0 - *aaa* instead of aaa*
- it('should return a subset of results ' + query3, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query3, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(config.numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.optionsProperty.should.match(/(\b|^)aaa(\b|$)/);
- });
- }
- done();
- });
- });
- var query4 = "where title contains 'tale'";
- it('should return a subset of results ' + query4, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query4, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(config.numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.title.should.match(/tale/i);
- });
- }
- done();
- });
- });
- var query5 = "where title contains 'ta*'";
- it('should return a subset of results ' + query5, function(done) {
- entities.getWithQuery(config.genericTestCollection1, query5, maxNumberOfEntities, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- if (body.entities.length > 0) {
- body.entities.length.should.be.greaterThan(0).and.lessThan(config.numberOfEntities + 1);
- body.entities.forEach(function(entity) {
- entity.title.should.match(/ta.*/i);
- });
- }
- done();
- });
- });
- var query6 = "where consistentProperty contains 'some*'";
- it('should return a subset of results ' + query6, function() {
- entities.getWithQuery('horses', query6, 10, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.be.greaterThan(0).and.lessThan(11);
- });
- });
- var query7 = "where consistentProperty contains 'ssccxxome*'";
- it('should not return a subset of results ' + query7, function() {
- var query = "where firstProperty contains 'ssccxxome*'";
- entities.getWithQuery('horses', query7, 10, function(err, body) {
- should(err).be.null;
- body.entities.should.be.an.instanceOf(Array);
- body.entities.length.should.be.equal(0);
-
- });
- });
- });
- }
-}
[33/50] [abbrv] usergrid git commit: add better exception handling
Posted by sn...@apache.org.
add better exception handling
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3cb0a0e0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3cb0a0e0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3cb0a0e0
Branch: refs/heads/asf-site
Commit: 3cb0a0e0c5a8a1f1e06d3bd71b9a2bcef333ca80
Parents: b485517
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Oct 9 08:39:27 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Oct 9 08:39:27 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 19 ++++++++++++++-----
1 file changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3cb0a0e0/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 0ac3860..b384b80 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -307,7 +307,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
});
//resolve the list and return it.
- final List<IndexEventResult> indexEventResults = masterObservable.toList().toBlocking().lastOrDefault(null);
+ final List<IndexEventResult> indexEventResults = masterObservable
+ .collect(() -> new ArrayList<IndexEventResult>(), (list,indexEventResult) -> list.add(indexEventResult) )
+ .toBlocking().lastOrDefault(null);
//if nothing came back then return null
if(indexEventResults==null){
return null;
@@ -318,13 +320,16 @@ public class AmazonAsyncEventService implements AsyncEventService {
//stream and filer the messages
List<QueueMessage> messagesToAck = indexEventResults.stream()
.map(indexEventResult -> {
+ //collect into the index submission
if (indexEventResult.getIndexOperationMessage().isPresent()) {
combined.ingest(indexEventResult.getIndexOperationMessage().get());
}
return indexEventResult;
})
+ //filter out the ones that need to be ack'd
.filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
.map(indexEventResult -> {
+ //record the cycle time
messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime());
return indexEventResult;
})
@@ -333,8 +338,13 @@ public class AmazonAsyncEventService implements AsyncEventService {
.collect(Collectors.toList());
//send the batch
- indexProducer.put(combined).toBlocking().lastOrDefault(null);
-
+ //TODO: should retry?
+ try {
+ indexProducer.put(combined).toBlocking().lastOrDefault(null);
+ }catch (Exception e){
+ logger.error("Failed to submit to index producer",messages,e);
+ throw e;
+ }
return messagesToAck;
}
@@ -584,10 +594,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
//ack each message, but only if we didn't error.
ack(messagesToAck);
- //messagesToAck.stream().forEach(message -> ack(message));
return messagesToAck;
} catch (Exception e) {
- logger.error("failed to ack messages to sqs", messages.get(0).getMessageId(), e);
+ logger.error("failed to ack messages to sqs", messages, e);
return null;
//do not rethrow so we can process all of them
}
[25/50] [abbrv] usergrid git commit: add configurable buffer size
Posted by sn...@apache.org.
add configurable buffer size
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d57b4fe2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d57b4fe2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d57b4fe2
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: d57b4fe2cc668f2b91bc26a83b216c846890c6b9
Parents: f9f0825
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 7 16:43:22 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 7 16:43:22 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/AmazonAsyncEventService.java | 4 ++--
.../apache/usergrid/corepersistence/index/IndexProcessorFig.java | 4 ++++
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d57b4fe2/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index a9e2459..957ee68 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -307,7 +307,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
//filter for success, send to the index(optional), ack
return masterObservable
//take the max
- .buffer(250, TimeUnit.MILLISECONDS, bufferSize)
+ .buffer(indexProcessorFig.getBufferTime(), TimeUnit.MILLISECONDS, bufferSize)
//map them to index results and return them
.flatMap(indexEventResults -> {
IndexOperationMessage combined = new IndexOperationMessage();
@@ -565,7 +565,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
{
final int bufferSize = messages.size();
return handleMessages(messages)
- .buffer(100, TimeUnit.MILLISECONDS, bufferSize) //TODO how to ack multiple messages via buffer
+ .buffer(indexProcessorFig.getBufferTime(), TimeUnit.MILLISECONDS, bufferSize) //TODO how to ack multiple messages via buffer
.doOnNext(messagesToAck -> {
if (messagesToAck.size() == 0) {
return;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d57b4fe2/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 2ffa374..410f162 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -90,6 +90,10 @@ public interface IndexProcessorFig extends GuicyFig {
@Key("elasticsearch.reindex.flush.interval")
int getUpdateInterval();
+ @Default("100")
+ @Key("elasticsearch.buffer.time_ms")
+ int getBufferTime();
+
@Default("1000")
@Key( REINDEX_BUFFER_SIZE )
int getReindexBufferSize();
[14/50] [abbrv] usergrid git commit: cleanup observable,
add realistic timer, make sure messages that error are not ack'd
Posted by sn...@apache.org.
cleanup observable, add realistic timer, make sure messages that error are not ack'd
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9da0179f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9da0179f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9da0179f
Branch: refs/heads/asf-site
Commit: 9da0179f17d636d44ee8c4121c909632582aafe7
Parents: 7c5a864
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 6 18:29:56 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 6 18:29:56 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 67 ++++++++++----------
1 file changed, 33 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9da0179f/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 1bc70cd..0d7553e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -262,36 +262,44 @@ public class AmazonAsyncEventService implements AsyncEventService {
}catch (ClassCastException cce){
logger.error("Failed to deserialize message body",cce);
}
- logger.debug("Processing {} event", event);
if (event == null) {
logger.error("AsyncEvent type or event is null!");
- return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent()));
+ return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(),System.currentTimeMillis()));
}
+
+ final AsyncEvent thisEvent = event;
+ if(logger.isDebugEnabled()) {
+ logger.debug("Processing {} event", event);
+ }
+
try {
+ Observable<IndexOperationMessage> indexoperationObservable;
//merge each operation to a master observable;
if (event instanceof EdgeDeleteEvent) {
- return handleIndexOperation(message, queueMessage -> handleEdgeDelete(queueMessage));
+ indexoperationObservable = handleEdgeDelete(message);
} else if (event instanceof EdgeIndexEvent) {
- return handleIndexOperation(message, queueMessage -> handleEdgeIndex(queueMessage));
+ indexoperationObservable = handleEdgeIndex(message);
} else if (event instanceof EntityDeleteEvent) {
- return handleIndexOperation(message, queueMessage -> handleEntityDelete(queueMessage));
+ indexoperationObservable = handleEntityDelete(message);
} else if (event instanceof EntityIndexEvent) {
- return handleIndexOperation(message, queueMessage -> handleEntityIndexUpdate(queueMessage));
+ indexoperationObservable = handleEntityIndexUpdate(message);
} else if (event instanceof InitializeApplicationIndexEvent) {
//does not return observable
- handleInitializeApplicationIndex(message);
- return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent()));
+ handleInitializeApplicationIndex(event,message);
+ indexoperationObservable = Observable.just(new IndexOperationMessage());
} else {
- logger.error("Unknown EventType: {}", event);//TODO: print json instead
- return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent()));
+ throw new Exception("Unknown EventType");//TODO: print json instead
}
- } catch (Exception e) {
- logger.error("Failed to index entity", e, message);
- return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent()));
- } finally {
- messageCycle.update(System.currentTimeMillis() - event.getCreationTime());
+ //return type that can be indexed and ack'd later
+ return indexoperationObservable
+ .map(indexOperationMessage ->
+ new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage),thisEvent.getCreationTime())
+ );
+ } catch (Exception e) {
+ logger.error("Failed to index message: " + message.getMessageId(), e, message);
+ return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime()));
}
});
@@ -315,25 +323,13 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
return indexProducer.put(combined)
.flatMap(indexOperationMessage -> Observable.from(indexEventResults))
+ .doOnNext(indexEventResult -> messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()))
.map(result -> result.getQueueMessage().get());
});
}
- //transform index operation to
- private Observable<IndexEventResult> handleIndexOperation(QueueMessage queueMessage,
- Func1<QueueMessage, Observable<IndexOperationMessage>> operation
- ){
- try{
- return operation.call(queueMessage)
- .map(indexOperationMessage -> new IndexEventResult(Optional.fromNullable(queueMessage), Optional.fromNullable(indexOperationMessage)));
- }catch (Exception e){
- logger.error("failed to run index",e);
- return Observable.just( new IndexEventResult(Optional.fromNullable(queueMessage), Optional.<IndexOperationMessage>absent()));
- }
- }
-
@Override
public void queueInitializeApplicationIndex( final ApplicationScope applicationScope) {
@@ -479,11 +475,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
- public void handleInitializeApplicationIndex(final QueueMessage message) {
+ public void handleInitializeApplicationIndex(final AsyncEvent event, final QueueMessage message) {
Preconditions.checkNotNull(message, "Queue Message cannot be null for handleInitializeApplicationIndex");
-
- final AsyncEvent event = (AsyncEvent) message.getBody();
- Preconditions.checkNotNull( message, "QueueMessage Body cannot be null for handleInitializeApplicationIndex" );
Preconditions.checkArgument(event instanceof InitializeApplicationIndexEvent, String.format("Event Type for handleInitializeApplicationIndex must be APPLICATION_INDEX, got %s", event.getClass()));
final InitializeApplicationIndexEvent initializeApplicationIndexEvent =
@@ -492,7 +485,6 @@ public class AmazonAsyncEventService implements AsyncEventService {
final IndexLocationStrategy indexLocationStrategy = initializeApplicationIndexEvent.getIndexLocationStrategy();
final EntityIndex index = entityIndexFactory.createEntityIndex( indexLocationStrategy );
index.initialize();
- ack( message );
}
/**
@@ -607,15 +599,18 @@ public class AmazonAsyncEventService implements AsyncEventService {
public class IndexEventResult{
private final Optional<QueueMessage> queueMessage;
private final Optional<IndexOperationMessage> indexOperationMessage;
+ private final long creationTime;
- public IndexEventResult(Optional<QueueMessage> queueMessage, Optional<IndexOperationMessage> indexOperationMessage ){
+ public IndexEventResult(Optional<QueueMessage> queueMessage, Optional<IndexOperationMessage> indexOperationMessage, long creationTime){
this.queueMessage = queueMessage;
this.indexOperationMessage = indexOperationMessage;
+ this.creationTime = creationTime;
}
+
public Optional<QueueMessage> getQueueMessage() {
return queueMessage;
}
@@ -623,5 +618,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
public Optional<IndexOperationMessage> getIndexOperationMessage() {
return indexOperationMessage;
}
+
+ public long getCreationTime() {
+ return creationTime;
+ }
}
}
[43/50] [abbrv] usergrid git commit: remove observable
Posted by sn...@apache.org.
remove observable
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/dbf37e48
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/dbf37e48
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/dbf37e48
Branch: refs/heads/asf-site
Commit: dbf37e48caee2ca7fb5a2468f4800ad4bdf196ea
Parents: 9cbe283
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 13 11:14:21 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 13 11:14:21 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 114 +++++++++----------
1 file changed, 57 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/dbf37e48/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index f1a02ce..95126c6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import com.google.common.base.Optional;
import org.apache.usergrid.persistence.index.impl.IndexProducer;
@@ -248,7 +249,6 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
-
/**
* calls the event handlers and returns a result with information on whether it needs to be ack'd and whether it needs to be indexed
* @param messages
@@ -259,71 +259,71 @@ public class AmazonAsyncEventService implements AsyncEventService {
logger.debug("callEventHandlers with {} message", messages.size());
}
- Observable<IndexEventResult> masterObservable = Observable.from(messages).map(message -> {
- AsyncEvent event = null;
- try {
- event = (AsyncEvent) message.getBody();
- } catch (ClassCastException cce) {
- logger.error("Failed to deserialize message body", cce);
- }
-
- if (event == null) {
- logger.error("AsyncEvent type or event is null!");
- return new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(), System.currentTimeMillis());
- }
-
- final AsyncEvent thisEvent = event;
- if (logger.isDebugEnabled()) {
- logger.debug("Processing {} event", event);
- }
-
- try {
- Observable<IndexOperationMessage> indexoperationObservable;
- //merge each operation to a master observable;
- if (event instanceof EdgeDeleteEvent) {
- indexoperationObservable = handleEdgeDelete(message);
- } else if (event instanceof EdgeIndexEvent) {
- indexoperationObservable = handleEdgeIndex(message);
- } else if (event instanceof EntityDeleteEvent) {
- indexoperationObservable = handleEntityDelete(message);
- } else if (event instanceof EntityIndexEvent) {
- indexoperationObservable = handleEntityIndexUpdate(message);
- } else if (event instanceof InitializeApplicationIndexEvent) {
- //does not return observable
- handleInitializeApplicationIndex(event, message);
- indexoperationObservable = Observable.just(new IndexOperationMessage());
- } else {
- throw new Exception("Unknown EventType");//TODO: print json instead
+ Stream<IndexEventResult> indexEventResults = messages.stream()
+ .map(message -> {
+ AsyncEvent event = null;
+ try {
+ event = (AsyncEvent) message.getBody();
+ } catch (ClassCastException cce) {
+ logger.error("Failed to deserialize message body", cce);
}
- //collect all of the
- IndexOperationMessage indexOperationMessage =
- indexoperationObservable
- .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
- .toBlocking().lastOrDefault(null);
+ if (event == null) {
+ logger.error("AsyncEvent type or event is null!");
+ return new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent(), System.currentTimeMillis());
+ }
- if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
- logger.info("Received empty index sequence message:({}), body:({}) ",
- message.getMessageId(),message.getStringBody());
+ final AsyncEvent thisEvent = event;
+ if (logger.isDebugEnabled()) {
+ logger.debug("Processing {} event", event);
}
- //return type that can be indexed and ack'd later
- return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
- } catch (Exception e) {
- logger.error("Failed to index message: " + message.getMessageId(), message.getStringBody() ,e);
- return new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime());
- }
- });
- //resolve the list and return it.
- final List<IndexEventResult> indexEventResults = masterObservable
- .collect(() -> new ArrayList<IndexEventResult>(), (list,indexEventResult) -> list.add(indexEventResult) )
- .toBlocking().lastOrDefault(null);
+ try {
+ //check for empty sets if this is true
+ boolean validateEmptySets = true;
+ Observable<IndexOperationMessage> indexoperationObservable;
+ //merge each operation to a master observable;
+ if (event instanceof EdgeDeleteEvent) {
+ indexoperationObservable = handleEdgeDelete(message);
+ } else if (event instanceof EdgeIndexEvent) {
+ indexoperationObservable = handleEdgeIndex(message);
+ } else if (event instanceof EntityDeleteEvent) {
+ indexoperationObservable = handleEntityDelete(message);
+ } else if (event instanceof EntityIndexEvent) {
+ indexoperationObservable = handleEntityIndexUpdate(message);
+ } else if (event instanceof InitializeApplicationIndexEvent) {
+ //does not return observable
+ handleInitializeApplicationIndex(event, message);
+ indexoperationObservable = Observable.just(new IndexOperationMessage());
+ validateEmptySets = false; //do not check this one for an empty set b/c it will be empty.
+ } else {
+ throw new Exception("Unknown EventType");//TODO: print json instead
+ }
+
+ //collect all of the
+ IndexOperationMessage indexOperationMessage =
+ indexoperationObservable
+ .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
+ .toBlocking().lastOrDefault(null);
+
+ if (validateEmptySets && (indexOperationMessage == null || indexOperationMessage.isEmpty())) {
+ logger.error("Received empty index sequence message:({}), body:({}) ",
+ message.getMessageId(), message.getStringBody());
+ throw new Exception("Received empty index sequence.");
+ }
+
+ //return type that can be indexed and ack'd later
+ return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
+ } catch (Exception e) {
+ logger.error("Failed to index message: " + message.getMessageId(), message.getStringBody(), e);
+ return new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent(), event.getCreationTime());
+ }
+ });
- return indexEventResults;
+ return indexEventResults.collect(Collectors.toList());
}
-
@Override
public void queueInitializeApplicationIndex( final ApplicationScope applicationScope) {
IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(
[22/50] [abbrv] usergrid git commit: buffer size fix
Posted by sn...@apache.org.
buffer size fix
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f9f08253
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f9f08253
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f9f08253
Branch: refs/heads/asf-site
Commit: f9f0825315c22ecfa1375189bf70020cde868661
Parents: 79caa09
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 7 15:14:22 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 7 15:14:22 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f9f08253/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index a5342ea..a9e2459 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -255,6 +255,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
logger.debug("handleMessages with {} message", messages.size());
}
+ final int bufferSize = messages.size();
Observable<IndexEventResult> masterObservable = Observable.from(messages).flatMap(message -> {
AsyncEvent event = null;
try{
@@ -305,10 +306,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
//filter for success, send to the index(optional), ack
return masterObservable
- //remove unsuccessful
- .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
//take the max
- .buffer( MAX_TAKE )
+ .buffer(250, TimeUnit.MILLISECONDS, bufferSize)
//map them to index results and return them
.flatMap(indexEventResults -> {
IndexOperationMessage combined = new IndexOperationMessage();
@@ -322,8 +321,13 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
return indexProducer.put(combined)
+ //change observable type
.flatMap(indexOperationMessage -> Observable.from(indexEventResults))
+ //remove unsuccessful
+ .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
+ //measure
.doOnNext(indexEventResult -> messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()))
+ //return the queue messages to ack
.map(result -> result.getQueueMessage().get());
});
[07/50] [abbrv] usergrid git commit: fix observable class cast
exception
Posted by sn...@apache.org.
fix observable class cast exception
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/890611de
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/890611de
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/890611de
Branch: refs/heads/asf-site
Commit: 890611de99bfce5ee02753168b101549d89d5205
Parents: d6bf2fa
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 5 18:46:43 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 5 18:46:43 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/AmazonAsyncEventService.java | 5 ++++-
.../usergrid/persistence/index/impl/EsIndexProducerImpl.java | 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/890611de/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index abb76c2..37a9da5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -446,7 +446,10 @@ public class AmazonAsyncEventService implements AsyncEventService {
entityDeleteResults = eventBuilder.buildEntityDelete( applicationScope, entityId );
- final Observable<IndexOperationMessage> merged = entityDeleteResults.getEntitiesCompacted().flatMap(mvccLogEntries -> entityDeleteResults.getIndexObservable()) ;
+ final Observable<IndexOperationMessage> merged = entityDeleteResults
+ .getEntitiesCompacted()
+ .collect(() -> new ArrayList<>(),(list,item)-> list.add(item))
+ .flatMap(collected -> entityDeleteResults.getIndexObservable()) ;
return merged;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/890611de/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index 828027c..409c2bc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -207,6 +207,7 @@ public class EsIndexProducerImpl implements IndexProducer {
if ( error ) {
if(errorString.lastIndexOf("rejected execution (queue capacity")>=0){
try{
+ log.warn("Encountered Queue Capacity Exception from ElasticSearch slowing by " +indexFig.getSleepTimeForQueueError());
Thread.sleep(indexFig.getSleepTimeForQueueError());
}catch (InterruptedException ie){
//move on
[26/50] [abbrv] usergrid git commit: add additional logging
Posted by sn...@apache.org.
add additional logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/365f6dc2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/365f6dc2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/365f6dc2
Branch: refs/heads/asf-site
Commit: 365f6dc2ac36720b221d043ef0c28503c2a9513c
Parents: d57b4fe
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 8 14:39:15 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 8 14:39:15 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 52 +++++++++++---------
1 file changed, 28 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/365f6dc2/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 957ee68..e3aca06 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -250,7 +250,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
- private Observable<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
+ private List<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
if (logger.isDebugEnabled()) {
logger.debug("handleMessages with {} message", messages.size());
}
@@ -305,9 +305,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
});
//filter for success, send to the index(optional), ack
- return masterObservable
+ return (List<QueueMessage>) masterObservable
//take the max
- .buffer(indexProcessorFig.getBufferTime(), TimeUnit.MILLISECONDS, bufferSize)
+ .buffer(bufferSize)
//map them to index results and return them
.flatMap(indexEventResults -> {
IndexOperationMessage combined = new IndexOperationMessage();
@@ -330,7 +330,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
//return the queue messages to ack
.map(result -> result.getQueueMessage().get());
- });
+ })
+ .doOnError(t -> logger.error("Failed to process queuemessages",t))
+ .toBlocking().lastOrDefault(null);
}
@@ -520,7 +522,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
private void startWorker() {
synchronized (mutex) {
- Observable<QueueMessage> consumer =
+ Observable<List<QueueMessage>> consumer =
Observable.create(new Observable.OnSubscribe<List<QueueMessage>>() {
@Override
public void call(final Subscriber<? super List<QueueMessage>> subscriber) {
@@ -561,26 +563,28 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
})
//this won't block our read loop, just reads and proceeds
- .flatMap(messages ->
- {
- final int bufferSize = messages.size();
- return handleMessages(messages)
- .buffer(indexProcessorFig.getBufferTime(), TimeUnit.MILLISECONDS, bufferSize) //TODO how to ack multiple messages via buffer
- .doOnNext(messagesToAck -> {
- if (messagesToAck.size() == 0) {
- return;
- }
- try {
- //ack each message, but only if we didn't error.
- ack(messagesToAck);
- } catch (Exception e) {
- logger.error("failed to ack messages to sqs", messagesToAck.get(0).getMessageId(), e);
- //do not rethrow so we can process all of them
- }
- })
- .flatMap(messagesToAck -> Observable.from(messagesToAck));
+ .map(messages ->
+ {
+ if (messages == null || messages.size() == 0) {
+ return null;
}
- );
+
+ try {
+
+ List<QueueMessage> messagesToAck = handleMessages(messages);
+
+ if (messagesToAck == null || messagesToAck.size() == 0) {
+ return messagesToAck;
+ }
+ //ack each message, but only if we didn't error.
+ ack(messagesToAck);
+ return messagesToAck;
+ } catch (Exception e) {
+ logger.error("failed to ack messages to sqs", messages.get(0).getMessageId(), e);
+ return null;
+ //do not rethrow so we can process all of them
+ }
+ });
//start in the background
[20/50] [abbrv] usergrid git commit: buffer size fix
Posted by sn...@apache.org.
buffer size fix
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/79caa09d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/79caa09d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/79caa09d
Branch: refs/heads/asf-site
Commit: 79caa09d49c1805e6e9379e82ce6f64cf043058d
Parents: f005ace
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 7 15:07:24 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 7 15:07:24 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/79caa09d/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 0d7553e..a5342ea 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -558,9 +558,14 @@ public class AmazonAsyncEventService implements AsyncEventService {
})
//this won't block our read loop, just reads and proceeds
.flatMap(messages ->
- handleMessages(messages)
- .buffer(MAX_TAKE)
+ {
+ final int bufferSize = messages.size();
+ return handleMessages(messages)
+ .buffer(100, TimeUnit.MILLISECONDS, bufferSize) //TODO how to ack multiple messages via buffer
.doOnNext(messagesToAck -> {
+ if (messagesToAck.size() == 0) {
+ return;
+ }
try {
//ack each message, but only if we didn't error.
ack(messagesToAck);
@@ -569,7 +574,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
//do not rethrow so we can process all of them
}
})
- .flatMap(messagesToAck -> Observable.from(messagesToAck))
+ .flatMap(messagesToAck -> Observable.from(messagesToAck));
+ }
);
//start in the background
[06/50] [abbrv] usergrid git commit: fix observable class cast
exception
Posted by sn...@apache.org.
fix observable class cast exception
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/890611de
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/890611de
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/890611de
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 890611de99bfce5ee02753168b101549d89d5205
Parents: d6bf2fa
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 5 18:46:43 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 5 18:46:43 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/AmazonAsyncEventService.java | 5 ++++-
.../usergrid/persistence/index/impl/EsIndexProducerImpl.java | 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/890611de/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index abb76c2..37a9da5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -446,7 +446,10 @@ public class AmazonAsyncEventService implements AsyncEventService {
entityDeleteResults = eventBuilder.buildEntityDelete( applicationScope, entityId );
- final Observable<IndexOperationMessage> merged = entityDeleteResults.getEntitiesCompacted().flatMap(mvccLogEntries -> entityDeleteResults.getIndexObservable()) ;
+ final Observable<IndexOperationMessage> merged = entityDeleteResults
+ .getEntitiesCompacted()
+ .collect(() -> new ArrayList<>(),(list,item)-> list.add(item))
+ .flatMap(collected -> entityDeleteResults.getIndexObservable()) ;
return merged;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/890611de/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index 828027c..409c2bc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -207,6 +207,7 @@ public class EsIndexProducerImpl implements IndexProducer {
if ( error ) {
if(errorString.lastIndexOf("rejected execution (queue capacity")>=0){
try{
+ log.warn("Encountered Queue Capacity Exception from ElasticSearch slowing by " +indexFig.getSleepTimeForQueueError());
Thread.sleep(indexFig.getSleepTimeForQueueError());
}catch (InterruptedException ie){
//move on
[38/50] [abbrv] usergrid git commit: add new exception for empty
sequences
Posted by sn...@apache.org.
add new exception for empty sequences
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a2a07aac
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a2a07aac
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a2a07aac
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: a2a07aac1c7f53b5bbaffe0a9bc433d05777cbdf
Parents: 5b65086
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 12 11:05:49 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 12 11:05:49 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/AmazonAsyncEventService.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a2a07aac/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 4ee2094..45d83cd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -302,6 +302,10 @@ public class AmazonAsyncEventService implements AsyncEventService {
.collect(() -> new IndexOperationMessage(), (collector, single ) -> collector.ingest(single))
.toBlocking().lastOrDefault(null);
+ if(indexOperationMessage == null){
+ throw new IllegalArgumentException("Received null index operation.");
+ }
+
//return type that can be indexed and ack'd later
return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
} catch (Exception e) {
[21/50] [abbrv] usergrid git commit: buffer size fix
Posted by sn...@apache.org.
buffer size fix
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/79caa09d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/79caa09d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/79caa09d
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 79caa09d49c1805e6e9379e82ce6f64cf043058d
Parents: f005ace
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 7 15:07:24 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 7 15:07:24 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/79caa09d/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 0d7553e..a5342ea 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -558,9 +558,14 @@ public class AmazonAsyncEventService implements AsyncEventService {
})
//this won't block our read loop, just reads and proceeds
.flatMap(messages ->
- handleMessages(messages)
- .buffer(MAX_TAKE)
+ {
+ final int bufferSize = messages.size();
+ return handleMessages(messages)
+ .buffer(100, TimeUnit.MILLISECONDS, bufferSize) //TODO how to ack multiple messages via buffer
.doOnNext(messagesToAck -> {
+ if (messagesToAck.size() == 0) {
+ return;
+ }
try {
//ack each message, but only if we didn't error.
ack(messagesToAck);
@@ -569,7 +574,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
//do not rethrow so we can process all of them
}
})
- .flatMap(messagesToAck -> Observable.from(messagesToAck))
+ .flatMap(messagesToAck -> Observable.from(messagesToAck));
+ }
);
//start in the background
[46/50] [abbrv] usergrid git commit: merge 2.1-release
Posted by sn...@apache.org.
merge 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f19ae087
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f19ae087
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f19ae087
Branch: refs/heads/asf-site
Commit: f19ae0870ac86fba312f5604d9c2a1474de65a88
Parents: 91fa489 0f589f6
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 14 15:50:07 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 14 15:50:07 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 22 +-
.../asyncevents/AmazonAsyncEventService.java | 280 ++++++++++++-------
.../asyncevents/EventBuilderImpl.java | 3 +-
.../asyncevents/InMemoryAsyncEventService.java | 7 +-
.../index/IndexProcessorFig.java | 4 +
.../index/ReIndexServiceImpl.java | 2 +-
.../migration/DeDupConnectionDataMigration.java | 2 +-
.../read/search/CandidateEntityFilter.java | 3 +-
.../pipeline/read/search/CandidateIdFilter.java | 2 +-
.../core/rx/ExceptionBehaviorTest.java | 171 +++++++++++
.../usergrid/persistence/index/IndexFig.java | 3 +
.../index/impl/EsIndexProducerImpl.java | 28 +-
.../EsIndexMappingMigrationPlugin.java | 2 +-
.../usergrid/persistence/queue/QueueFig.java | 12 +-
.../persistence/queue/QueueMessage.java | 10 +
.../queue/impl/SNSQueueManagerImpl.java | 2 +
.../queue/impl/SQSQueueManagerImpl.java | 1 +
.../queue/util/AmazonNotificationUtils.java | 1 +
.../rest/system/ApplicationsResource.java | 28 +-
stack/rest_integration_tests/README.md | 19 --
stack/rest_integration_tests/config/default.js | 38 ---
stack/rest_integration_tests/config/index.js | 25 --
stack/rest_integration_tests/index.js | 20 --
stack/rest_integration_tests/lib/connections.js | 137 ---------
stack/rest_integration_tests/lib/entities.js | 166 -----------
.../rest_integration_tests/lib/notifications.js | 68 -----
stack/rest_integration_tests/lib/random.js | 74 -----
stack/rest_integration_tests/lib/response.js | 30 --
stack/rest_integration_tests/lib/token.js | 49 ----
stack/rest_integration_tests/lib/urls.js | 37 ---
stack/rest_integration_tests/lib/users.js | 82 ------
stack/rest_integration_tests/package.json | 18 --
.../test/authentication/management.js | 38 ---
.../test/authentication/org.js | 37 ---
.../test/authentication/resetPassword.js | 78 ------
.../test/authentication/user.js | 40 ---
.../test/connections/create.js | 49 ----
.../test/connections/delete.js | 77 -----
.../test/connections/get.js | 82 ------
.../test/entities/create.js | 60 ----
.../test/entities/deleteAll.js | 37 ---
.../rest_integration_tests/test/entities/get.js | 51 ----
.../test/entities/update.js | 43 ---
stack/rest_integration_tests/test/main.js | 72 -----
stack/rest_integration_tests/test/mocha.opts | 3 -
.../test/notifications/create.js | 36 ---
.../test/queries/comparison.js | 58 ----
.../test/queries/contains.js | 116 --------
.../test/queries/equals.js | 54 ----
.../test/queries/location.js | 42 ---
.../test/queries/order.js | 82 ------
stack/rest_integration_tests/test/setup.js | 103 -------
stack/rest_integration_tests/test/teardown.js | 65 -----
.../rest_integration_tests/test/users/create.js | 45 ---
.../management/AppInfoMigrationPlugin.java | 2 +-
.../services/AbstractConnectionsService.java | 2 +-
56 files changed, 452 insertions(+), 2166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index f1e6ad6,aad7610..1bd5c7d
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -525,11 -527,16 +526,12 @@@ public class CpRelationManager implemen
*
*/
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
- final EntityIndexBatch batch = ei.createBatch();
-
- // remove item from collection index
- SearchEdge indexScope = createCollectionSearchEdge( cpHeadEntity.getId(), collectionName );
- batch.deindex( indexScope, memberEntity );
+ //TODO: this should not happen here, needs to go to SQS
+ //indexProducer.put(batch).subscribe();
+ indexService.queueEntityDelete(applicationScope,memberEntity.getId());
- managerCache.getIndexProducer().put( batch.build()).toBlocking().lastOrDefault(null); // this should throw an exception
+
// special handling for roles collection of a group
if ( headEntity.getType().equals( Group.ENTITY_TYPE ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
index 82bc4c8,0000000..55887c7
mode 100644,000000..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
@@@ -1,168 -1,0 +1,178 @@@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.rest.system;
+
+import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Classy class class.
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class ApplicationsResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger(ApplicationsResource.class);
+
+
+ public ApplicationsResource() {
+ logger.info( "ApplicationsResource initialized" );
+ }
+
+ @RequireSystemAccess
+ @DELETE
+ @JSONP
+ @Path( "{applicationId}" )
+ public ApiResponse clearApplication(
+ @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @QueryParam( "confirmApplicationName" ) String confirmApplicationName,
+ @QueryParam( "limit" ) int limit,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+
+ throws Exception {
+
+ if(confirmApplicationName == null){
+ throw new IllegalArgumentException("please make add a QueryString for confirmApplicationName");
+ }
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final EntityManager em = emf.getEntityManager(applicationId);
+ final String name = em.getApplication().getApplicationName();
+ if(!name.toLowerCase().equals(confirmApplicationName.toLowerCase())){
+ throw new IllegalArgumentException(
+ "confirmApplicationName: " + confirmApplicationName + " does not equal " + name);
+ }
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ logger.info("clearing up application");
+
+ final Thread delete = new Thread() {
+
+ @Override
+ public void run() {
+ final AtomicInteger itemsDeleted = new AtomicInteger(0);
+ try {
+ management.deleteAllEntities(applicationId, limit)
+ .map(id -> itemsDeleted.incrementAndGet())
+ .doOnNext(count -> {
+ if( count % 100 == 0 ){
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("count",itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map).subscribe();
++ statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map)
++ .subscribe();//do not want to throw this exception
+ }
+ })
- .doOnCompleted(() ->{
- Map<String,Object> map = new LinkedHashMap<>();
- map.put("count",itemsDeleted.intValue());
++ .doOnCompleted(() -> {
++ Map<String, Object> map = new LinkedHashMap<>();
++ map.put("count", itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId,jobId, StatusService.Status.COMPLETE,map).subscribe();
++ statusService.setStatus(applicationId, jobId, StatusService.Status.COMPLETE, map)
++ .toBlocking().lastOrDefault(null);//want to rethrow this exception
+ })
- .subscribe();
++ .toBlocking().lastOrDefault(null);//expecting exception to be caught if job fails
+
+ } catch ( Exception e ) {
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("exception",e);
- statusService.setStatus(applicationId,jobId, StatusService.Status.FAILED,map).subscribe();
++ try {
++ statusService.setStatus(applicationId, jobId, StatusService.Status.FAILED, map).toBlocking().lastOrDefault(null);//leave as subscribe if fails retry
++ }catch (Exception subE){
++ logger.error("failed to update status "+jobId,subE);
++ }
+ logger.error( "Failed to delete appid:"+applicationId + " jobid:"+jobId+" count:"+itemsDeleted, e );
+ }
+ }
+ };
+
+ delete.setName("Delete for app : " + applicationId + " job: " + jobId);
+ delete.setDaemon(true);
+ delete.start();
+
- statusService.setStatus(applicationId,jobId, StatusService.Status.STARTED,new LinkedHashMap<>()).subscribe();
-
++ try {
++ //should throw exception if can't start
++ statusService.setStatus(applicationId, jobId, StatusService.Status.STARTED, new LinkedHashMap<>()).toBlocking().lastOrDefault(null);
++ }catch (Exception e){
++ logger.error("failed to set status for " + jobId, e);
++ }
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put("status",StatusService.Status.STARTED);
+ response.setData(data);
+ response.setSuccess();
+ return response;
+ }
+
+ @RequireSystemAccess
+ @GET
+ @Path( "{applicationId}/job/{jobId}" )
+ public ApiResponse getStatus(
+ @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @PathParam("jobId") UUID jobId,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception{
+
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ StatusService.JobStatus jobStatus = statusService.getStatus(applicationId, jobId).toBlocking().lastOrDefault(null);
+
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put( "status", jobStatus.getStatus().toString() );
+ data.put( "metadata", jobStatus.getData() );
+ response.setData(data);
+ response.setSuccess();
+ return response;
+ }
+
+}
[24/50] [abbrv] usergrid git commit: add configurable buffer size
Posted by sn...@apache.org.
add configurable buffer size
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d57b4fe2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d57b4fe2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d57b4fe2
Branch: refs/heads/asf-site
Commit: d57b4fe2cc668f2b91bc26a83b216c846890c6b9
Parents: f9f0825
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 7 16:43:22 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 7 16:43:22 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/AmazonAsyncEventService.java | 4 ++--
.../apache/usergrid/corepersistence/index/IndexProcessorFig.java | 4 ++++
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d57b4fe2/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index a9e2459..957ee68 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -307,7 +307,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
//filter for success, send to the index(optional), ack
return masterObservable
//take the max
- .buffer(250, TimeUnit.MILLISECONDS, bufferSize)
+ .buffer(indexProcessorFig.getBufferTime(), TimeUnit.MILLISECONDS, bufferSize)
//map them to index results and return them
.flatMap(indexEventResults -> {
IndexOperationMessage combined = new IndexOperationMessage();
@@ -565,7 +565,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
{
final int bufferSize = messages.size();
return handleMessages(messages)
- .buffer(100, TimeUnit.MILLISECONDS, bufferSize) //TODO how to ack multiple messages via buffer
+ .buffer(indexProcessorFig.getBufferTime(), TimeUnit.MILLISECONDS, bufferSize) //TODO how to ack multiple messages via buffer
.doOnNext(messagesToAck -> {
if (messagesToAck.size() == 0) {
return;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d57b4fe2/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 2ffa374..410f162 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -90,6 +90,10 @@ public interface IndexProcessorFig extends GuicyFig {
@Key("elasticsearch.reindex.flush.interval")
int getUpdateInterval();
+ @Default("100")
+ @Key("elasticsearch.buffer.time_ms")
+ int getBufferTime();
+
@Default("1000")
@Key( REINDEX_BUFFER_SIZE )
int getReindexBufferSize();
[48/50] [abbrv] usergrid git commit: Merge branch 'master' into
usergrid-1007-shiro-cache
Posted by sn...@apache.org.
Merge branch 'master' into usergrid-1007-shiro-cache
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/341b4736
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/341b4736
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/341b4736
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 341b47365936e824a0f12972fa17710b4176a3dc
Parents: 5427631 f19ae08
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Oct 15 08:59:00 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Oct 15 08:59:00 2015 -0400
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 22 +-
.../asyncevents/AmazonAsyncEventService.java | 280 ++++++++++++-------
.../asyncevents/EventBuilderImpl.java | 3 +-
.../asyncevents/InMemoryAsyncEventService.java | 7 +-
.../index/IndexProcessorFig.java | 4 +
.../index/ReIndexServiceImpl.java | 2 +-
.../migration/DeDupConnectionDataMigration.java | 2 +-
.../read/search/CandidateEntityFilter.java | 3 +-
.../pipeline/read/search/CandidateIdFilter.java | 2 +-
.../core/rx/ExceptionBehaviorTest.java | 171 +++++++++++
.../usergrid/persistence/index/IndexFig.java | 3 +
.../index/impl/EsIndexProducerImpl.java | 28 +-
.../EsIndexMappingMigrationPlugin.java | 2 +-
.../usergrid/persistence/queue/QueueFig.java | 12 +-
.../persistence/queue/QueueMessage.java | 10 +
.../queue/impl/SNSQueueManagerImpl.java | 2 +
.../queue/impl/SQSQueueManagerImpl.java | 1 +
.../queue/util/AmazonNotificationUtils.java | 1 +
.../rest/system/ApplicationsResource.java | 28 +-
stack/rest_integration_tests/README.md | 19 --
stack/rest_integration_tests/config/default.js | 38 ---
stack/rest_integration_tests/config/index.js | 25 --
stack/rest_integration_tests/index.js | 20 --
stack/rest_integration_tests/lib/connections.js | 137 ---------
stack/rest_integration_tests/lib/entities.js | 166 -----------
.../rest_integration_tests/lib/notifications.js | 68 -----
stack/rest_integration_tests/lib/random.js | 74 -----
stack/rest_integration_tests/lib/response.js | 30 --
stack/rest_integration_tests/lib/token.js | 49 ----
stack/rest_integration_tests/lib/urls.js | 37 ---
stack/rest_integration_tests/lib/users.js | 82 ------
stack/rest_integration_tests/package.json | 18 --
.../test/authentication/management.js | 38 ---
.../test/authentication/org.js | 37 ---
.../test/authentication/resetPassword.js | 78 ------
.../test/authentication/user.js | 40 ---
.../test/connections/create.js | 49 ----
.../test/connections/delete.js | 77 -----
.../test/connections/get.js | 82 ------
.../test/entities/create.js | 60 ----
.../test/entities/deleteAll.js | 37 ---
.../rest_integration_tests/test/entities/get.js | 51 ----
.../test/entities/update.js | 43 ---
stack/rest_integration_tests/test/main.js | 72 -----
stack/rest_integration_tests/test/mocha.opts | 3 -
.../test/notifications/create.js | 36 ---
.../test/queries/comparison.js | 58 ----
.../test/queries/contains.js | 116 --------
.../test/queries/equals.js | 54 ----
.../test/queries/location.js | 42 ---
.../test/queries/order.js | 82 ------
stack/rest_integration_tests/test/setup.js | 103 -------
stack/rest_integration_tests/test/teardown.js | 65 -----
.../rest_integration_tests/test/users/create.js | 45 ---
.../management/AppInfoMigrationPlugin.java | 2 +-
.../services/AbstractConnectionsService.java | 2 +-
56 files changed, 452 insertions(+), 2166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/341b4736/stack/services/src/main/java/org/apache/usergrid/services/AbstractConnectionsService.java
----------------------------------------------------------------------
[02/50] [abbrv] usergrid git commit: fix subscribe for messages
Posted by sn...@apache.org.
fix subscribe for messages
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1a1d42e1
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1a1d42e1
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1a1d42e1
Branch: refs/heads/asf-site
Commit: 1a1d42e1f53cabf433442c17f614f9fcae418a22
Parents: b437f61
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 5 16:28:04 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 5 16:28:04 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1a1d42e1/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index e215d48..bf29c5a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -285,13 +285,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
- return indexProducer.put( combined )
- .flatMap( operationResult -> Observable.from( indexEventResults ) )
- //ack each message, but only if we didn't error. If we did, we'll want to log it and
- .map( indexEventResult -> {
- ack( indexEventResult.queueMessage );
- return indexEventResult;
- } );
+ return indexProducer.put(combined)
+ .flatMap(operationResult -> Observable.from(indexEventResults));
+
} );
}
@@ -538,7 +534,15 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
})
//this won't block our read loop, just reads and proceeds
- .flatMap( messages -> handleMessages( messages ) ).subscribeOn( Schedulers.newThread() );
+ .map(messages ->
+ handleMessages(messages)
+ .map(indexEventResult -> {
+ ack( indexEventResult.getQueueMessage() );
+ return indexEventResult;
+ })
+ .toBlocking().lastOrDefault(null)
+ )//ack each message, but only if we didn't error. If we did, we'll want to log it and
+ .subscribeOn( Schedulers.newThread() );
//start in the background
[13/50] [abbrv] usergrid git commit: commit stash
Posted by sn...@apache.org.
commit stash
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7c5a864d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7c5a864d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7c5a864d
Branch: refs/heads/asf-site
Commit: 7c5a864dd1ce22ada5456ced1a6c10f9a5533b1f
Parents: 7dceb56
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 6 14:34:33 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 6 14:34:33 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 71 ++++++++++++++------
.../index/impl/EsIndexProducerImpl.java | 6 +-
2 files changed, 53 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7c5a864d/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 14d37b5..1bc70cd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.base.Optional;
@@ -226,6 +227,28 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
+ /**
+ * Ack message in SQS
+ */
+ public void ack(final List<QueueMessage> messages) {
+
+ final Timer.Context timer = this.ackTimer.time();
+
+ try{
+ queue.commitMessages(messages);
+
+ //decrement our in-flight counter
+ inFlight.decrementAndGet();
+
+ }catch(Exception e){
+ throw new RuntimeException("Unable to ack messages", e);
+ }finally {
+ timer.stop();
+ }
+
+
+ }
+
private Observable<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
if (logger.isDebugEnabled()) {
@@ -243,7 +266,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
if (event == null) {
logger.error("AsyncEvent type or event is null!");
- return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), true));
+ return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent()));
}
try {
//merge each operation to a master observable;
@@ -258,23 +281,24 @@ public class AmazonAsyncEventService implements AsyncEventService {
} else if (event instanceof InitializeApplicationIndexEvent) {
//does not return observable
handleInitializeApplicationIndex(message);
- return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), false));
+ return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent()));
} else {
- logger.error("Unknown EventType: {}", event);
- return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), true));
+ logger.error("Unknown EventType: {}", event);//TODO: print json instead
+ return Observable.just(new IndexEventResult(Optional.fromNullable(message), Optional.<IndexOperationMessage>absent()));
}
} catch (Exception e) {
logger.error("Failed to index entity", e, message);
- return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), false));
+ return Observable.just(new IndexEventResult(Optional.absent(), Optional.<IndexOperationMessage>absent()));
} finally {
messageCycle.update(System.currentTimeMillis() - event.getCreationTime());
}
});
+ //filter for success, send to the index(optional), ack
return masterObservable
//remove unsuccessful
- .filter(indexEventResult -> indexEventResult.shouldProcess())
+ .filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
//take the max
.buffer( MAX_TAKE )
//map them to index results and return them
@@ -290,8 +314,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
return indexProducer.put(combined)
- .flatMap(operationResult -> Observable.from(indexEventResults))
- .map(result -> result.getQueueMessage());
+ .flatMap(indexOperationMessage -> Observable.from(indexEventResults))
+ .map(result -> result.getQueueMessage().get());
});
@@ -303,10 +327,10 @@ public class AmazonAsyncEventService implements AsyncEventService {
){
try{
return operation.call(queueMessage)
- .map(indexOperationMessage -> new IndexEventResult(queueMessage, Optional.fromNullable(indexOperationMessage), true));
+ .map(indexOperationMessage -> new IndexEventResult(Optional.fromNullable(queueMessage), Optional.fromNullable(indexOperationMessage)));
}catch (Exception e){
logger.error("failed to run index",e);
- return Observable.just( new IndexEventResult(queueMessage, Optional.<IndexOperationMessage>absent(),false));
+ return Observable.just( new IndexEventResult(Optional.fromNullable(queueMessage), Optional.<IndexOperationMessage>absent()));
}
}
@@ -543,10 +567,17 @@ public class AmazonAsyncEventService implements AsyncEventService {
//this won't block our read loop, just reads and proceeds
.flatMap(messages ->
handleMessages(messages)
- .doOnNext(message -> {
- //ack each message, but only if we didn't error.
- ack(message);
+ .buffer(MAX_TAKE)
+ .doOnNext(messagesToAck -> {
+ try {
+ //ack each message, but only if we didn't error.
+ ack(messagesToAck);
+ } catch (Exception e) {
+ logger.error("failed to ack messages to sqs", messagesToAck.get(0).getMessageId(), e);
+ //do not rethrow so we can process all of them
+ }
})
+ .flatMap(messagesToAck -> Observable.from(messagesToAck))
);
//start in the background
@@ -574,23 +605,19 @@ public class AmazonAsyncEventService implements AsyncEventService {
public class IndexEventResult{
- private final QueueMessage queueMessage;
+ private final Optional<QueueMessage> queueMessage;
private final Optional<IndexOperationMessage> indexOperationMessage;
- private final boolean shouldProcess;
- public IndexEventResult(QueueMessage queueMessage, Optional<IndexOperationMessage> indexOperationMessage ,boolean shouldProcess){
+
+ public IndexEventResult(Optional<QueueMessage> queueMessage, Optional<IndexOperationMessage> indexOperationMessage ){
this.queueMessage = queueMessage;
this.indexOperationMessage = indexOperationMessage;
- this.shouldProcess = shouldProcess;
- }
- public QueueMessage getQueueMessage() {
- return queueMessage;
}
- public boolean shouldProcess() {
- return shouldProcess;
+ public Optional<QueueMessage> getQueueMessage() {
+ return queueMessage;
}
public Optional<IndexOperationMessage> getIndexOperationMessage() {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7c5a864d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index 409c2bc..e778948 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -115,6 +115,7 @@ public class EsIndexProducerImpl implements IndexProducer {
final Observable<IndexOperation> index = Observable.from(batch.getIndexRequests());
final Observable<DeIndexOperation> deIndex = Observable.from(batch.getDeIndexRequests());
+ //TODO: look at indexing ordering
final Observable<BatchOperation> batchOps = Observable.merge(index, deIndex);
//buffer into the max size we can send ES and fire them all off until we're completed
@@ -207,9 +208,10 @@ public class EsIndexProducerImpl implements IndexProducer {
if ( error ) {
if(errorString.lastIndexOf("rejected execution (queue capacity")>=0){
try{
- log.warn("Encountered Queue Capacity Exception from ElasticSearch slowing by " +indexFig.getSleepTimeForQueueError());
+ log.warn("Encountered Queue Capacity Exception from ElasticSearch slowing by "
+ + indexFig.getSleepTimeForQueueError() );
Thread.sleep(indexFig.getSleepTimeForQueueError());
- }catch (InterruptedException ie){
+ }catch (Exception e){
//move on
}
}
[32/50] [abbrv] usergrid git commit: add better exception handling
Posted by sn...@apache.org.
add better exception handling
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3cb0a0e0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3cb0a0e0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3cb0a0e0
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 3cb0a0e0c5a8a1f1e06d3bd71b9a2bcef333ca80
Parents: b485517
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Oct 9 08:39:27 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Oct 9 08:39:27 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 19 ++++++++++++++-----
1 file changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3cb0a0e0/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 0ac3860..b384b80 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -307,7 +307,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
});
//resolve the list and return it.
- final List<IndexEventResult> indexEventResults = masterObservable.toList().toBlocking().lastOrDefault(null);
+ final List<IndexEventResult> indexEventResults = masterObservable
+ .collect(() -> new ArrayList<IndexEventResult>(), (list,indexEventResult) -> list.add(indexEventResult) )
+ .toBlocking().lastOrDefault(null);
//if nothing came back then return null
if(indexEventResults==null){
return null;
@@ -318,13 +320,16 @@ public class AmazonAsyncEventService implements AsyncEventService {
//stream and filer the messages
List<QueueMessage> messagesToAck = indexEventResults.stream()
.map(indexEventResult -> {
+ //collect into the index submission
if (indexEventResult.getIndexOperationMessage().isPresent()) {
combined.ingest(indexEventResult.getIndexOperationMessage().get());
}
return indexEventResult;
})
+ //filter out the ones that need to be ack'd
.filter(indexEventResult -> indexEventResult.getQueueMessage().isPresent())
.map(indexEventResult -> {
+ //record the cycle time
messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime());
return indexEventResult;
})
@@ -333,8 +338,13 @@ public class AmazonAsyncEventService implements AsyncEventService {
.collect(Collectors.toList());
//send the batch
- indexProducer.put(combined).toBlocking().lastOrDefault(null);
-
+ //TODO: should retry?
+ try {
+ indexProducer.put(combined).toBlocking().lastOrDefault(null);
+ }catch (Exception e){
+ logger.error("Failed to submit to index producer",messages,e);
+ throw e;
+ }
return messagesToAck;
}
@@ -584,10 +594,9 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
//ack each message, but only if we didn't error.
ack(messagesToAck);
- //messagesToAck.stream().forEach(message -> ack(message));
return messagesToAck;
} catch (Exception e) {
- logger.error("failed to ack messages to sqs", messages.get(0).getMessageId(), e);
+ logger.error("failed to ack messages to sqs", messages, e);
return null;
//do not rethrow so we can process all of them
}
[08/50] [abbrv] usergrid git commit: refactor observable
Posted by sn...@apache.org.
refactor observable
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/17586ecb
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/17586ecb
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/17586ecb
Branch: refs/heads/usergrid-1007-shiro-cache
Commit: 17586ecb5dd38ee71b7edfd256fdecb50e615556
Parents: 890611d
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue Oct 6 08:26:46 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue Oct 6 08:26:46 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/17586ecb/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 37a9da5..e16de05 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -227,7 +227,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
- private Observable<IndexEventResult> handleMessages( final List<QueueMessage> messages ) {
+ private Observable<QueueMessage> handleMessages( final List<QueueMessage> messages ) {
if (logger.isDebugEnabled()) {
logger.debug("handleMessages with {} message", messages.size());
}
@@ -290,7 +290,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
//ack after successful completion of the operation.
return indexProducer.put(combined)
- .flatMap(operationResult -> Observable.from(indexEventResults));
+ .flatMap(operationResult -> Observable.from(indexEventResults))
+ .map(result -> result.getQueueMessage());
});
@@ -448,7 +449,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
final Observable<IndexOperationMessage> merged = entityDeleteResults
.getEntitiesCompacted()
- .collect(() -> new ArrayList<>(),(list,item)-> list.add(item))
+ .collect(() -> new ArrayList<>(), (list, item) -> list.add(item))
.flatMap(collected -> entityDeleteResults.getIndexObservable()) ;
return merged;
}
@@ -499,7 +500,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
private void startWorker() {
synchronized (mutex) {
- Observable<IndexEventResult> consumer =
+ Observable<QueueMessage> consumer =
Observable.create(new Observable.OnSubscribe<List<QueueMessage>>() {
@Override
public void call(final Subscriber<? super List<QueueMessage>> subscriber) {
@@ -540,14 +541,13 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
})
//this won't block our read loop, just reads and proceeds
- .map(messages ->
+ .flatMap(messages ->
handleMessages(messages)
- .map(indexEventResult -> {
- ack(indexEventResult.getQueueMessage());
- return indexEventResult;
+ .doOnNext(message -> {
+ //ack each message, but only if we didn't error.
+ ack(message);
})
- .toBlocking().lastOrDefault(null)
- )//ack each message, but only if we didn't error. If we did, we'll want to log it and
+ )
.subscribeOn(Schedulers.newThread());
//start in the background
[04/50] [abbrv] usergrid git commit: fix observable class cast
exception
Posted by sn...@apache.org.
fix observable class cast exception
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d6bf2fac
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d6bf2fac
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d6bf2fac
Branch: refs/heads/asf-site
Commit: d6bf2fac1b5b7ee994d13c7efce007d1f0dc6de2
Parents: 1a1d42e
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 5 18:36:05 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 5 18:36:05 2015 -0600
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 37 +++++++++++---------
1 file changed, 20 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d6bf2fac/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index bf29c5a..abb76c2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -233,8 +233,12 @@ public class AmazonAsyncEventService implements AsyncEventService {
}
Observable<IndexEventResult> masterObservable = Observable.from(messages).flatMap(message -> {
- final AsyncEvent event = (AsyncEvent) message.getBody();
-
+ AsyncEvent event = null;
+ try{
+ event = (AsyncEvent) message.getBody();
+ }catch (ClassCastException cce){
+ logger.error("Failed to deserialize message body",cce);
+ }
logger.debug("Processing {} event", event);
if (event == null) {
@@ -259,10 +263,10 @@ public class AmazonAsyncEventService implements AsyncEventService {
logger.error("Unknown EventType: {}", event);
return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), true));
}
- }catch (Exception e){
- logger.error("Failed to index entity", e,message);
+ } catch (Exception e) {
+ logger.error("Failed to index entity", e, message);
return Observable.just(new IndexEventResult(message, Optional.<IndexOperationMessage>absent(), false));
- }finally {
+ } finally {
messageCycle.update(System.currentTimeMillis() - event.getCreationTime());
}
@@ -270,25 +274,25 @@ public class AmazonAsyncEventService implements AsyncEventService {
return masterObservable
//remove unsuccessful
- .filter( indexEventResult -> indexEventResult.shouldProcess() )
+ .filter(indexEventResult -> indexEventResult.shouldProcess())
//take the max
.buffer( MAX_TAKE )
//map them to index results and return them
- .flatMap( indexEventResults -> {
+ .flatMap(indexEventResults -> {
IndexOperationMessage combined = new IndexOperationMessage();
indexEventResults.stream().forEach(
- indexEventResult ->{
- if(indexEventResult.getIndexOperationMessage().isPresent()) {
+ indexEventResult -> {
+ if (indexEventResult.getIndexOperationMessage().isPresent()) {
combined.ingest(indexEventResult.getIndexOperationMessage().get());
}
- } );
+ });
//ack after successful completion of the operation.
return indexProducer.put(combined)
.flatMap(operationResult -> Observable.from(indexEventResults));
- } );
+ });
}
@@ -376,8 +380,8 @@ public class AmazonAsyncEventService implements AsyncEventService {
final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
- final Observable<IndexOperationMessage> edgeIndexObservable = ecm.load(edgeIndexEvent.getEntityId()).flatMap( entity -> eventBuilder.buildNewEdge(
- applicationScope, entity, edge ) );
+ final Observable<IndexOperationMessage> edgeIndexObservable = ecm.load(edgeIndexEvent.getEntityId()).flatMap(entity -> eventBuilder.buildNewEdge(
+ applicationScope, entity, edge));
return edgeIndexObservable;
}
@@ -442,8 +446,7 @@ public class AmazonAsyncEventService implements AsyncEventService {
entityDeleteResults = eventBuilder.buildEntityDelete( applicationScope, entityId );
- final Observable merged = Observable.merge( entityDeleteResults.getEntitiesCompacted(),
- entityDeleteResults.getIndexObservable() );
+ final Observable<IndexOperationMessage> merged = entityDeleteResults.getEntitiesCompacted().flatMap(mvccLogEntries -> entityDeleteResults.getIndexObservable()) ;
return merged;
}
@@ -537,12 +540,12 @@ public class AmazonAsyncEventService implements AsyncEventService {
.map(messages ->
handleMessages(messages)
.map(indexEventResult -> {
- ack( indexEventResult.getQueueMessage() );
+ ack(indexEventResult.getQueueMessage());
return indexEventResult;
})
.toBlocking().lastOrDefault(null)
)//ack each message, but only if we didn't error. If we did, we'll want to log it and
- .subscribeOn( Schedulers.newThread() );
+ .subscribeOn(Schedulers.newThread());
//start in the background
[39/50] [abbrv] usergrid git commit: add new exception for empty
sequences
Posted by sn...@apache.org.
add new exception for empty sequences
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a2a07aac
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a2a07aac
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a2a07aac
Branch: refs/heads/asf-site
Commit: a2a07aac1c7f53b5bbaffe0a9bc433d05777cbdf
Parents: 5b65086
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Oct 12 11:05:49 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Oct 12 11:05:49 2015 -0600
----------------------------------------------------------------------
.../corepersistence/asyncevents/AmazonAsyncEventService.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a2a07aac/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
index 4ee2094..45d83cd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AmazonAsyncEventService.java
@@ -302,6 +302,10 @@ public class AmazonAsyncEventService implements AsyncEventService {
.collect(() -> new IndexOperationMessage(), (collector, single ) -> collector.ingest(single))
.toBlocking().lastOrDefault(null);
+ if(indexOperationMessage == null){
+ throw new IllegalArgumentException("Received null index operation.");
+ }
+
//return type that can be indexed and ack'd later
return new IndexEventResult(Optional.fromNullable(message), Optional.fromNullable(indexOperationMessage), thisEvent.getCreationTime());
} catch (Exception e) {