You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/10/15 00:07:23 UTC
[1/3] usergrid git commit: Lowers the default viability timeout to 10
seconds
Repository: usergrid
Updated Branches:
refs/heads/remove-inmemory-event-service d536b5b4e -> 5b5fbf03e
Lowers the default viability timeout to 10 seconds
Changes retry count to 20. During this, we should hit the 10% read repair chance if we're missing data.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0f589f68
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0f589f68
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0f589f68
Branch: refs/heads/remove-inmemory-event-service
Commit: 0f589f688ee72fe9db2b50e7af6604152cd898ff
Parents: dbf37e4
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Oct 14 14:27:42 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Oct 14 14:27:42 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/queue/QueueFig.java | 12 ++++++++++--
.../persistence/queue/util/AmazonNotificationUtils.java | 1 +
2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0f589f68/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
index 66f8af4..7f26bcf 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
@@ -56,10 +56,10 @@ public interface QueueFig extends GuicyFig {
String getDeadletterRetentionPeriod();
/**
- * The maximum number of messages to deliver to a dead letter queue.
+ * The maximum number of attempts to attempt to deliver before failing into the DLQ
*/
@Key( "usergrid.queue.deliveryLimit" )
- @Default("5")
+ @Default("20")
String getQueueDeliveryLimit();
@Key("usergrid.use.default.queue")
@@ -74,4 +74,12 @@ public interface QueueFig extends GuicyFig {
@Key("usergrid.queue.publish.queuesize")
@Default("850000")
int getAsyncQueueSize();
+
+ /**
+ * Set the visibility timeout for faster retries
+ * @return
+ */
+ @Key( "usergrid.queue.visibilityTimeout" )
+ @Default("10")
+ String getVisibilityTimeout();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0f589f68/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
index c321715..6105592 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
@@ -64,6 +64,7 @@ public class AmazonNotificationUtils {
final Map<String, String> queueAttributes = new HashMap<>( 2 );
queueAttributes.put( "MessageRetentionPeriod", fig.getRetentionPeriod() );
queueAttributes.put( "RedrivePolicy", redrivePolicy );
+ queueAttributes.put( "VisibilityTimeout", fig.getVisibilityTimeout() );
CreateQueueRequest createQueueRequest = new CreateQueueRequest().
withQueueName( queueName )
[3/3] usergrid git commit: Merge branch 'master' into
remove-inmemory-event-service
Posted by sf...@apache.org.
Merge branch 'master' into remove-inmemory-event-service
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5b5fbf03
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5b5fbf03
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5b5fbf03
Branch: refs/heads/remove-inmemory-event-service
Commit: 5b5fbf03e735191784e3e8c6af4cc644eb60963a
Parents: d536b5b f19ae08
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 14 16:07:09 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 14 16:07:09 2015 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/queue/QueueFig.java | 12 ++++++++++--
.../persistence/queue/util/AmazonNotificationUtils.java | 1 +
2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[2/3] usergrid git commit: merge 2.1-release
Posted by sf...@apache.org.
merge 2.1-release
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f19ae087
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f19ae087
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f19ae087
Branch: refs/heads/remove-inmemory-event-service
Commit: f19ae0870ac86fba312f5604d9c2a1474de65a88
Parents: 91fa489 0f589f6
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Oct 14 15:50:07 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Oct 14 15:50:07 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 22 +-
.../asyncevents/AmazonAsyncEventService.java | 280 ++++++++++++-------
.../asyncevents/EventBuilderImpl.java | 3 +-
.../asyncevents/InMemoryAsyncEventService.java | 7 +-
.../index/IndexProcessorFig.java | 4 +
.../index/ReIndexServiceImpl.java | 2 +-
.../migration/DeDupConnectionDataMigration.java | 2 +-
.../read/search/CandidateEntityFilter.java | 3 +-
.../pipeline/read/search/CandidateIdFilter.java | 2 +-
.../core/rx/ExceptionBehaviorTest.java | 171 +++++++++++
.../usergrid/persistence/index/IndexFig.java | 3 +
.../index/impl/EsIndexProducerImpl.java | 28 +-
.../EsIndexMappingMigrationPlugin.java | 2 +-
.../usergrid/persistence/queue/QueueFig.java | 12 +-
.../persistence/queue/QueueMessage.java | 10 +
.../queue/impl/SNSQueueManagerImpl.java | 2 +
.../queue/impl/SQSQueueManagerImpl.java | 1 +
.../queue/util/AmazonNotificationUtils.java | 1 +
.../rest/system/ApplicationsResource.java | 28 +-
stack/rest_integration_tests/README.md | 19 --
stack/rest_integration_tests/config/default.js | 38 ---
stack/rest_integration_tests/config/index.js | 25 --
stack/rest_integration_tests/index.js | 20 --
stack/rest_integration_tests/lib/connections.js | 137 ---------
stack/rest_integration_tests/lib/entities.js | 166 -----------
.../rest_integration_tests/lib/notifications.js | 68 -----
stack/rest_integration_tests/lib/random.js | 74 -----
stack/rest_integration_tests/lib/response.js | 30 --
stack/rest_integration_tests/lib/token.js | 49 ----
stack/rest_integration_tests/lib/urls.js | 37 ---
stack/rest_integration_tests/lib/users.js | 82 ------
stack/rest_integration_tests/package.json | 18 --
.../test/authentication/management.js | 38 ---
.../test/authentication/org.js | 37 ---
.../test/authentication/resetPassword.js | 78 ------
.../test/authentication/user.js | 40 ---
.../test/connections/create.js | 49 ----
.../test/connections/delete.js | 77 -----
.../test/connections/get.js | 82 ------
.../test/entities/create.js | 60 ----
.../test/entities/deleteAll.js | 37 ---
.../rest_integration_tests/test/entities/get.js | 51 ----
.../test/entities/update.js | 43 ---
stack/rest_integration_tests/test/main.js | 72 -----
stack/rest_integration_tests/test/mocha.opts | 3 -
.../test/notifications/create.js | 36 ---
.../test/queries/comparison.js | 58 ----
.../test/queries/contains.js | 116 --------
.../test/queries/equals.js | 54 ----
.../test/queries/location.js | 42 ---
.../test/queries/order.js | 82 ------
stack/rest_integration_tests/test/setup.js | 103 -------
stack/rest_integration_tests/test/teardown.js | 65 -----
.../rest_integration_tests/test/users/create.js | 45 ---
.../management/AppInfoMigrationPlugin.java | 2 +-
.../services/AbstractConnectionsService.java | 2 +-
56 files changed, 452 insertions(+), 2166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index f1e6ad6,aad7610..1bd5c7d
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -525,11 -527,16 +526,12 @@@ public class CpRelationManager implemen
*
*/
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
- final EntityIndexBatch batch = ei.createBatch();
-
- // remove item from collection index
- SearchEdge indexScope = createCollectionSearchEdge( cpHeadEntity.getId(), collectionName );
- batch.deindex( indexScope, memberEntity );
+ //TODO: this should not happen here, needs to go to SQS
+ //indexProducer.put(batch).subscribe();
+ indexService.queueEntityDelete(applicationScope,memberEntity.getId());
- managerCache.getIndexProducer().put( batch.build()).toBlocking().lastOrDefault(null); // this should throw an exception
+
// special handling for roles collection of a group
if ( headEntity.getType().equals( Group.ENTITY_TYPE ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f19ae087/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
index 82bc4c8,0000000..55887c7
mode 100644,000000..100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/ApplicationsResource.java
@@@ -1,168 -1,0 +1,178 @@@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.rest.system;
+
+import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+import org.apache.usergrid.corepersistence.service.StatusService;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.rest.AbstractContextResource;
+import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.annotation.Scope;
+import org.springframework.stereotype.Component;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.UriInfo;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Classy class class.
+ */
+@Component
+@Scope( "singleton" )
+@Produces( {
+ MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
+ "application/ecmascript", "text/jscript"
+} )
+public class ApplicationsResource extends AbstractContextResource {
+
+ private static final Logger logger = LoggerFactory.getLogger(ApplicationsResource.class);
+
+
+ public ApplicationsResource() {
+ logger.info( "ApplicationsResource initialized" );
+ }
+
+ @RequireSystemAccess
+ @DELETE
+ @JSONP
+ @Path( "{applicationId}" )
+ public ApiResponse clearApplication(
+ @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @QueryParam( "confirmApplicationName" ) String confirmApplicationName,
+ @QueryParam( "limit" ) int limit,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+
+ throws Exception {
+
+ if(confirmApplicationName == null){
+ throw new IllegalArgumentException("please make add a QueryString for confirmApplicationName");
+ }
+
+ final UUID jobId = UUIDGenerator.newTimeUUID();
+
+ final EntityManager em = emf.getEntityManager(applicationId);
+ final String name = em.getApplication().getApplicationName();
+ if(!name.toLowerCase().equals(confirmApplicationName.toLowerCase())){
+ throw new IllegalArgumentException(
+ "confirmApplicationName: " + confirmApplicationName + " does not equal " + name);
+ }
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ logger.info("clearing up application");
+
+ final Thread delete = new Thread() {
+
+ @Override
+ public void run() {
+ final AtomicInteger itemsDeleted = new AtomicInteger(0);
+ try {
+ management.deleteAllEntities(applicationId, limit)
+ .map(id -> itemsDeleted.incrementAndGet())
+ .doOnNext(count -> {
+ if( count % 100 == 0 ){
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("count",itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map).subscribe();
++ statusService.setStatus(applicationId, jobId, StatusService.Status.INPROGRESS,map)
++ .subscribe();//do not want to throw this exception
+ }
+ })
- .doOnCompleted(() ->{
- Map<String,Object> map = new LinkedHashMap<>();
- map.put("count",itemsDeleted.intValue());
++ .doOnCompleted(() -> {
++ Map<String, Object> map = new LinkedHashMap<>();
++ map.put("count", itemsDeleted.intValue());
+ final StatusService statusService = injector.getInstance(StatusService.class);
- statusService.setStatus(applicationId,jobId, StatusService.Status.COMPLETE,map).subscribe();
++ statusService.setStatus(applicationId, jobId, StatusService.Status.COMPLETE, map)
++ .toBlocking().lastOrDefault(null);//want to rethrow this exception
+ })
- .subscribe();
++ .toBlocking().lastOrDefault(null);//expecting exception to be caught if job fails
+
+ } catch ( Exception e ) {
+ Map<String,Object> map = new LinkedHashMap<>();
+ map.put("exception",e);
- statusService.setStatus(applicationId,jobId, StatusService.Status.FAILED,map).subscribe();
++ try {
++ statusService.setStatus(applicationId, jobId, StatusService.Status.FAILED, map).toBlocking().lastOrDefault(null);//leave as subscribe if fails retry
++ }catch (Exception subE){
++ logger.error("failed to update status "+jobId,subE);
++ }
+ logger.error( "Failed to delete appid:"+applicationId + " jobid:"+jobId+" count:"+itemsDeleted, e );
+ }
+ }
+ };
+
+ delete.setName("Delete for app : " + applicationId + " job: " + jobId);
+ delete.setDaemon(true);
+ delete.start();
+
- statusService.setStatus(applicationId,jobId, StatusService.Status.STARTED,new LinkedHashMap<>()).subscribe();
-
++ try {
++ //should throw exception if can't start
++ statusService.setStatus(applicationId, jobId, StatusService.Status.STARTED, new LinkedHashMap<>()).toBlocking().lastOrDefault(null);
++ }catch (Exception e){
++ logger.error("failed to set status for " + jobId, e);
++ }
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put("status",StatusService.Status.STARTED);
+ response.setData(data);
+ response.setSuccess();
+ return response;
+ }
+
+ @RequireSystemAccess
+ @GET
+ @Path( "{applicationId}/job/{jobId}" )
+ public ApiResponse getStatus(
+ @Context UriInfo ui,
+ @PathParam("applicationId") UUID applicationId,
+ @PathParam("jobId") UUID jobId,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback ) throws Exception{
+
+ final StatusService statusService = injector.getInstance(StatusService.class);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear application" );
+
+ StatusService.JobStatus jobStatus = statusService.getStatus(applicationId, jobId).toBlocking().lastOrDefault(null);
+
+ Map<String,Object> data = new HashMap<>();
+ data.put("jobId",jobId);
+ data.put( "status", jobStatus.getStatus().toString() );
+ data.put( "metadata", jobStatus.getData() );
+ response.setData(data);
+ response.setSuccess();
+ return response;
+ }
+
+}