You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/08/19 23:17:21 UTC
[1/3] usergrid git commit: Reduce the cache size for graph shards.
Repository: usergrid
Updated Branches:
refs/heads/master 80e9f37d6 -> d2061dc3d
Reduce the cache size for graph shards.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6a54d0db
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6a54d0db
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6a54d0db
Branch: refs/heads/master
Commit: 6a54d0db03b4a990a44d3afe6255e4b7d2ce716e
Parents: f9196037
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Aug 19 10:26:31 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Aug 19 10:26:31 2016 -0700
----------------------------------------------------------------------
.../usergrid/persistence/graph/GraphFig.java | 2 +-
.../impl/shard/impl/NodeShardCacheImpl.java | 46 +++++---------------
2 files changed, 13 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6a54d0db/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
index efd94ed..46f6f0c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
@@ -134,7 +134,7 @@ public interface GraphFig extends GuicyFig {
long getShardMinDelta();
- @Default("250000")
+ @Default("100000")
@Key(SHARD_CACHE_SIZE)
long getShardCacheSize();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6a54d0db/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
index 545ac37..0a259f0 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
@@ -72,18 +72,8 @@ import com.google.inject.Inject;
@Singleton
public class NodeShardCacheImpl implements NodeShardCache {
- /**
- * Only cache shards that have < 10k groups. This is an arbitrary amount, and may change with profiling and
- * testing
- */
- private static final int MAX_WEIGHT_PER_ELEMENT = 10000;
-
-
private final NodeShardAllocation nodeShardAllocation;
private final GraphFig graphFig;
-
-
-
private ListeningScheduledExecutorService refreshExecutors;
private LoadingCache<CacheKey, CacheEntry> graphs;
@@ -111,8 +101,8 @@ public class NodeShardCacheImpl implements NodeShardCache {
final String propertyName = evt.getPropertyName();
if ( propertyName.equals( GraphFig.SHARD_CACHE_SIZE ) || propertyName
- .equals( GraphFig.SHARD_CACHE_TIMEOUT ) || propertyName
- .equals( GraphFig.SHARD_CACHE_REFRESH_WORKERS ) ) {
+ .equals( GraphFig.SHARD_CACHE_TIMEOUT ) || propertyName
+ .equals( GraphFig.SHARD_CACHE_REFRESH_WORKERS ) ) {
updateCache();
@@ -206,20 +196,20 @@ public class NodeShardCacheImpl implements NodeShardCache {
}
this.refreshExecutors = MoreExecutors
- .listeningDecorator( Executors.newScheduledThreadPool( graphFig.getShardCacheRefreshWorkerCount() ) );
+ .listeningDecorator( Executors.newScheduledThreadPool( graphFig.getShardCacheRefreshWorkerCount() ) );
this.graphs = CacheBuilder.newBuilder()
- //we want to asynchronously load new values for existing ones, that way we wont' have to
- //wait for a trip to cassandra
- .refreshAfterWrite( graphFig.getShardCacheTimeout(), TimeUnit.MILLISECONDS )
+ //we want to asynchronously load new values for existing ones, that way we wont' have to
+ //wait for a trip to cassandra
+ .refreshAfterWrite( graphFig.getShardCacheTimeout(), TimeUnit.MILLISECONDS )
- //set our weight function, since not all shards are equal
- .maximumWeight(MAX_WEIGHT_PER_ELEMENT * graphFig.getShardCacheSize() ).weigher( new ShardWeigher() )
+ //set a static cache entry size here
+ .maximumSize(graphFig.getShardCacheSize())
- //set our shard loader
- .build( new ShardCacheLoader() );
+ //set our shard loader
+ .build( new ShardCacheLoader() );
}
@@ -282,7 +272,7 @@ public class NodeShardCacheImpl implements NodeShardCache {
private CacheEntry( final Iterator<ShardEntryGroup> shards ) {
Preconditions.checkArgument( shards.hasNext(),
- "More than 1 entry must be present in the shard to load into cache" );
+ "More than 1 entry must be present in the shard to load into cache" );
this.shards = new TreeMap<>();
/**
@@ -340,7 +330,7 @@ public class NodeShardCacheImpl implements NodeShardCache {
final Iterator<ShardEntryGroup> edges =
- nodeShardAllocation.getShards( key.scope, Optional.<Shard>absent(), key.directedEdgeMeta );
+ nodeShardAllocation.getShards( key.scope, Optional.<Shard>absent(), key.directedEdgeMeta );
final CacheEntry cacheEntry = new CacheEntry( edges );
@@ -363,16 +353,4 @@ public class NodeShardCacheImpl implements NodeShardCache {
//TODO, use RX for sliding window buffering and duplicate removal
}
-
-
- /**
- * Calculates the weight of the entry by geting the size of the cache
- */
- final class ShardWeigher implements Weigher<CacheKey, CacheEntry> {
-
- @Override
- public int weigh( final CacheKey key, final CacheEntry value ) {
- return value.getCacheSize();
- }
- }
}
[2/3] usergrid git commit: Enhance the SQS polling timeout and clean
up some log statements.
Posted by mr...@apache.org.
Enhance the SQS polling timeout and clean up some log statements.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b5ed32ac
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b5ed32ac
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b5ed32ac
Branch: refs/heads/master
Commit: b5ed32acc89f3a31d29f08b46afc16ddc01e87a2
Parents: 6a54d0d
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Aug 19 13:45:24 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Aug 19 13:45:24 2016 -0700
----------------------------------------------------------------------
.../usergrid/mq/cassandra/io/AbstractSearch.java | 8 +++++---
.../apache/usergrid/persistence/queue/QueueFig.java | 8 ++++----
.../persistence/queue/impl/SNSQueueManagerImpl.java | 14 +++++++++++---
.../usergrid/rest/applications/ServiceResource.java | 7 ++++++-
4 files changed, 26 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5ed32ac/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
index a1d36a9..4e23923 100644
--- a/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
+++ b/stack/core/src/main/java/org/apache/usergrid/mq/cassandra/io/AbstractSearch.java
@@ -151,7 +151,7 @@ public abstract class AbstractSearch implements QueueSearch {
if ( bounds == null ) {
logger.error( "Necessary queue bounds not found" );
- throw new QueueException( "Neccessary queue bounds not found" );
+ throw new QueueException( "Necessary queue bounds not found" );
}
UUID finish_uuid = params.reversed ? bounds.getOldest() : bounds.getNewest();
@@ -202,8 +202,10 @@ public abstract class AbstractSearch implements QueueSearch {
while ( ( current_ts_shard >= start_ts_shard ) && ( current_ts_shard <= finish_ts_shard )
&& comparator.compare( start, finish_uuid ) < 1 ) {
- logger.info( "Starting search with start UUID {}, finish UUID {}, and reversed {}",
- lastValue, finish_uuid, params.reversed );
+ if( logger.isDebugEnabled() ) {
+ logger.debug("Starting search with start UUID {}, finish UUID {}, and reversed {}",
+ lastValue, finish_uuid, params.reversed);
+ }
SliceQuery<ByteBuffer, UUID, ByteBuffer> q = createSliceQuery( ko, be, ue, be );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5ed32ac/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
index 6265a33..74912ae 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/QueueFig.java
@@ -92,12 +92,12 @@ public interface QueueFig extends GuicyFig {
int getQueueClientConnectionTimeout();
@Key( "usergrid.queue.client.socket.timeout")
- @Default( "20000" ) // 20 seconds
+ @Default( "50000" ) // 50 seconds
int getQueueClientSocketTimeout();
- @Key( "usergrid.queue.poll.timeshift")
- @Default( "3000" ) // 3 seconds
- int getQueuePollTimeshift();
+ @Key( "usergrid.queue.poll.timeout")
+ @Default( "10000" ) // 10 seconds
+ int getQueuePollTimeout();
@Key( "usergrid.queue.quorum.fallback")
@Default("false") // 30 seconds
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5ed32ac/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index 0be5bd0..51c6fbf 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -437,9 +437,17 @@ public class SNSQueueManagerImpl implements QueueManager {
receiveMessageRequest.setVisibilityTimeout(
Math.max( MIN_VISIBILITY_TIMEOUT, fig.getVisibilityTimeout() / 1000 ) );
- // set SQS long polling to 3 secs < the client socket timeout (network delays) with min of 0 (no long poll)
- receiveMessageRequest.setWaitTimeSeconds(
- Math.max(0, ( fig.getQueueClientSocketTimeout() - fig.getQueuePollTimeshift() ) / 1000 ) );
+
+ int longPollTimeout = Math.min(20000, fig.getQueuePollTimeout()); // 20000 is the SQS maximum
+
+ // ensure the client's socket timeout is not less than the configure long poll timeout
+ if( fig.getQueueClientSocketTimeout() < longPollTimeout){
+
+ longPollTimeout = Math.max(0, fig.getQueueClientSocketTimeout() - 1000);
+
+ }
+
+ receiveMessageRequest.setWaitTimeSeconds( longPollTimeout / 1000 ); // convert to seconds
try {
ReceiveMessageResult result = sqs.receiveMessage( receiveMessageRequest );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b5ed32ac/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
index 05b4aa7..91c7db9 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ServiceResource.java
@@ -944,7 +944,12 @@ public class ServiceResource extends AbstractContextResource {
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
}
catch(AmazonServiceException ase){
- logger.error(ase.getMessage());
+
+ if( ase.getStatusCode() > 499 ){
+ logger.error(ase.getMessage());
+ }else if(logger.isDebugEnabled()){
+ logger.debug(ase.getMessage());
+ }
return Response.status(ase.getStatusCode()).build();
}
catch(RuntimeException re){
[3/3] usergrid git commit: Merge branch 'hotfix-20160819'
Posted by mr...@apache.org.
Merge branch 'hotfix-20160819'
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d2061dc3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d2061dc3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d2061dc3
Branch: refs/heads/master
Commit: d2061dc3d1e0f6b279415fc0b09fac209323ca8a
Parents: 80e9f37 b5ed32a
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Aug 19 13:49:34 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Aug 19 13:49:34 2016 -0700
----------------------------------------------------------------------
.../mq/cassandra/io/AbstractSearch.java | 8 +++---
.../impl/shard/impl/NodeShardCacheImpl.java | 26 +++++++++++---------
.../usergrid/persistence/queue/QueueFig.java | 8 +++---
.../queue/impl/SNSQueueManagerImpl.java | 14 ++++++++---
.../rest/applications/ServiceResource.java | 7 +++++-
5 files changed, 40 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2061dc3/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
index ee9602c,0a259f0..30beaf9
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
@@@ -201,15 -201,15 +201,17 @@@ public class NodeShardCacheImpl impleme
this.graphs = CacheBuilder.newBuilder()
- //we want to asynchronously load new values for existing ones, that way we wont' have to
- //wait for a trip to cassandra
- .refreshAfterWrite( graphFig.getShardCacheTimeout(), TimeUnit.MILLISECONDS )
+ //we want to asynchronously load new values for existing ones, that way we wont' have to
+ //wait for a trip to cassandra
+ .refreshAfterWrite( graphFig.getShardCacheTimeout(), TimeUnit.MILLISECONDS )
- //set a static cache entry size here
- .maximumSize(graphFig.getShardCacheSize())
+
- //set our shard loader
- .build( new ShardCacheLoader() );
+ //set a static cache entry size here
+ .maximumSize(graphFig.getShardCacheSize())
+
+ //set our shard loader
+ .build( new ShardCacheLoader() );
++
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d2061dc3/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
----------------------------------------------------------------------