You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by md...@apache.org on 2017/12/13 19:52:22 UTC
[1/4] usergrid git commit: Change swapped config items
Repository: usergrid
Updated Branches:
refs/heads/master c172133e3 -> a88a87527
Change swapped config items
usergrid.cluster.region* - Akka cluster region config
usergrid.queue.region* - SQS region config
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/cda4d207
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/cda4d207
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/cda4d207
Branch: refs/heads/master
Commit: cda4d207ae71753a388e258f8d46d2d476adfa87
Parents: 4a65910
Author: Mike Dunker <md...@google.com>
Authored: Tue Dec 5 18:11:22 2017 -0800
Committer: Mike Dunker <md...@google.com>
Committed: Tue Dec 5 18:11:22 2017 -0800
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/EntityManagerFig.java | 2 +-
.../apache/usergrid/persistence/queue/LegacyQueueFig.java | 8 ++++----
.../usergrid/persistence/queue/impl/SNSQueueManagerImpl.java | 4 ++--
3 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cda4d207/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 46c7a1d..3c8a53f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@ -46,7 +46,7 @@ public interface EntityManagerFig extends GuicyFig {
* Comma-separated list of one or more Amazon regions to use if multiregion
* is set to true.
*/
- @Key( "usergrid.queue.regionList" )
+ @Key( "usergrid.cluster.region.list" )
@Default("us-east-1")
String getRegionList();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cda4d207/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
index f19bede..c399636 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
@@ -14,14 +14,14 @@ public interface LegacyQueueFig extends GuicyFig {
* http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html*
*/
- String USERGRID_CLUSTER_REGION_LIST = "usergrid.cluster.region.list";
- String USERGRID_CLUSTER_REGION_LOCAL = "usergrid.cluster.region.local";
+ String USERGRID_QUEUE_REGION_LIST = "usergrid.queue.regionList";
+ String USERGRID_QUEUE_REGION_LOCAL = "usergrid.queue.region";
/**
* Primary region to use for Amazon queues.
*/
- @Key( USERGRID_CLUSTER_REGION_LOCAL )
+ @Key(USERGRID_QUEUE_REGION_LOCAL)
@Default("us-east-1")
String getPrimaryRegion();
@@ -37,7 +37,7 @@ public interface LegacyQueueFig extends GuicyFig {
* Comma-separated list of one or more Amazon regions to use if multiregion
* is set to true.
*/
- @Key( USERGRID_CLUSTER_REGION_LIST )
+ @Key(USERGRID_QUEUE_REGION_LIST)
@Default("us-east-1")
String getRegionList();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/cda4d207/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index bc9be57..b5d52dc 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -248,7 +248,7 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
region = Region.getRegion(regions);
}
catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("INVALID REGION FROM CONFIGURATION " + LegacyQueueFig.USERGRID_CLUSTER_REGION_LIST + ": " + regionName, e);
+ throw new IllegalArgumentException("INVALID REGION FROM CONFIGURATION " + LegacyQueueFig.USERGRID_QUEUE_REGION_LIST + ": " + regionName, e);
}
AmazonSQSClient sqsClient = createSQSClient( region );
@@ -825,7 +825,7 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
return Region.getRegion(regions);
}
catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("INVALID PRIMARY REGION FROM CONFIGURATION " + LegacyQueueFig.USERGRID_CLUSTER_REGION_LOCAL + ": " + regionName, e);
+ throw new IllegalArgumentException("INVALID PRIMARY REGION FROM CONFIGURATION " + LegacyQueueFig.USERGRID_QUEUE_REGION_LOCAL + ": " + regionName, e);
}
}
[3/4] usergrid git commit: update SNS topic permissions for SQS
queues when necessary
Posted by md...@apache.org.
update SNS topic permissions for SQS queues when necessary
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2b357337
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2b357337
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2b357337
Branch: refs/heads/master
Commit: 2b3573377a96eda1341fd61568195476744aabd0
Parents: 71169f8
Author: Mike Dunker <md...@google.com>
Authored: Mon Dec 11 14:49:57 2017 -0800
Committer: Mike Dunker <md...@google.com>
Committed: Mon Dec 11 14:49:57 2017 -0800
----------------------------------------------------------------------
.../queue/util/AmazonNotificationUtils.java | 135 ++++++++++++++-----
1 file changed, 101 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2b357337/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
index 56bef91..b2b209c 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/util/AmazonNotificationUtils.java
@@ -1,21 +1,16 @@
package org.apache.usergrid.persistence.queue.util;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import com.amazonaws.auth.policy.*;
+import com.amazonaws.auth.policy.conditions.ArnCondition;
+import com.amazonaws.services.sqs.model.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.queue.LegacyQueueFig;
-import com.amazonaws.auth.policy.Condition;
-import com.amazonaws.auth.policy.Policy;
-import com.amazonaws.auth.policy.Principal;
-import com.amazonaws.auth.policy.Resource;
-import com.amazonaws.auth.policy.Statement;
import com.amazonaws.auth.policy.actions.SQSActions;
import com.amazonaws.auth.policy.conditions.ConditionFactory;
import com.amazonaws.services.sns.AmazonSNSClient;
@@ -23,13 +18,6 @@ import com.amazonaws.services.sns.model.CreateTopicResult;
import com.amazonaws.services.sns.model.ListTopicsResult;
import com.amazonaws.services.sns.model.Topic;
import com.amazonaws.services.sqs.AmazonSQSClient;
-import com.amazonaws.services.sqs.model.CreateQueueRequest;
-import com.amazonaws.services.sqs.model.CreateQueueResult;
-import com.amazonaws.services.sqs.model.GetQueueAttributesRequest;
-import com.amazonaws.services.sqs.model.GetQueueAttributesResult;
-import com.amazonaws.services.sqs.model.GetQueueUrlResult;
-import com.amazonaws.services.sqs.model.QueueDoesNotExistException;
-import com.amazonaws.services.sqs.model.SetQueueAttributesRequest;
/**
@@ -85,34 +73,113 @@ public class AmazonNotificationUtils {
public static void setQueuePermissionsToReceive( final AmazonSQSClient sqs, final String queueUrl,
final List<String> topicARNs ) throws Exception {
- String queueARN = getQueueArnByUrl( sqs, queueUrl );
-
- Statement statement = new Statement( Statement.Effect.Allow ).withActions( SQSActions.SendMessage )
- .withPrincipals( new Principal( "*" ) )
- .withResources( new Resource( queueARN ) );
+ // retrieve queue ARN and policy
+ List<String> sqsAttrNames = Arrays.asList(QueueAttributeName.QueueArn.toString(),
+ QueueAttributeName.Policy.toString());
+ GetQueueAttributesRequest getQueueAttributesRequest =
+ new GetQueueAttributesRequest( queueUrl ).withAttributeNames( sqsAttrNames );
+ GetQueueAttributesResult queueAttributesResult = sqs.getQueueAttributes( getQueueAttributesRequest );
+ Map<String, String> sqsAttributeMap = queueAttributesResult.getAttributes();
+ String queueARN = sqsAttributeMap.get(QueueAttributeName.QueueArn.toString());
+ String policyJson = sqsAttributeMap.get(QueueAttributeName.Policy.toString());
+
+ // cannot send ARN in settings update, so remove it
+ sqsAttributeMap.remove(QueueAttributeName.QueueArn.toString());
+
+ // get existing policy from JSON
+ Policy policy = policyJson != null && policyJson.length() > 0 ? Policy.fromJson(policyJson) : new Policy();
+
+ // see if permissions already exist, and find ArnLike conditions
+ boolean matchingConditionFound = false;
+ boolean policyEdited = false;
+ for (Statement statement : policy.getStatements()) {
+ logger.info("statement id: {}, effect: {}, action: {}, resources:{}",
+ statement.getId(), statement.getEffect().name(),
+ statement.getActions().get(0).getActionName(),
+ statement.getResources().get(0).getId());
+
+ // must be Allow effect
+ if (! statement.getEffect().name().equals(Statement.Effect.Allow.name())) {
+ continue;
+ }
- List<Condition> conditions = new ArrayList<>();
+ // must be SendMessage action
+ boolean actionFound = false;
+ for (Action action : statement.getActions()) {
+ // do lower case comparison, since UI adds SQS.SendMessage but SDK uses sqs.SendMessage
+ if (action.getActionName().toLowerCase().equals(SQSActions.SendMessage.getActionName().toLowerCase())) {
+ actionFound = true;
+ break;
+ }
+ }
+ if (!actionFound) {
+ continue;
+ }
- for ( String topicARN : topicARNs ) {
+ // must be same queue resource
+ boolean queueResourceFound = false;
+ for (Resource resource : statement.getResources()) {
+ if (resource.getId().equals(queueARN)) {
+ queueResourceFound = true;
+ break;
+ }
+ }
+ if (!queueResourceFound) {
+ continue;
+ }
- conditions.add( ConditionFactory.newSourceArnCondition( topicARN ) );
+ // found matching statement, check conditions for source ARN
+ for (Condition condition : statement.getConditions()) {
+ if (logger.isTraceEnabled()) {
+ logger.trace("condition type: {}, conditionKey: {}", condition.getType(), condition.getConditionKey());
+ }
+ if (condition.getType().equals(ArnCondition.ArnComparisonType.ArnLike.name()) &&
+ condition.getConditionKey().equals(ConditionFactory.SOURCE_ARN_CONDITION_KEY)) {
+ matchingConditionFound = true;
+ for (String topicARN : topicARNs) {
+ if (! condition.getValues().contains(topicARN)) {
+ // topic doesn't exist, add it
+ policyEdited = true;
+ condition.getValues().add(topicARN);
+ }
+ }
+ }
+ }
}
- statement.setConditions( conditions );
- Policy policy = new Policy( "SubscriptionPermission" ).withStatements( statement );
+ if (!matchingConditionFound) {
+ // never found ArnLike SourceArn condition, need to add a statement
+ List<Condition> conditions = new ArrayList<>();
+ for (String topicARN : topicARNs) {
- final Map<String, String> queueAttributes = new HashMap<>();
- queueAttributes.put( "Policy", policy.toJson() );
+ conditions.add(ConditionFactory.newSourceArnCondition(topicARN));
+ }
- SetQueueAttributesRequest queueAttributesRequest = new SetQueueAttributesRequest( queueUrl, queueAttributes );
+ Statement statement = new Statement(Statement.Effect.Allow)
+ .withPrincipals(Principal.AllUsers)
+ .withActions(SQSActions.SendMessage)
+ .withResources(new Resource(queueARN));
+ statement.setConditions(conditions);
- try {
- sqs.setQueueAttributes( queueAttributesRequest );
+ policy.getStatements().add(statement);
+ policyEdited = true;
}
- catch ( Exception e ) {
- logger.error( "Failed to set permissions on QUEUE ARN=[{}] for TOPIC ARNs=[{}]", queueARN,
- topicARNs.toString(), e );
+
+ if (policyEdited) {
+ sqsAttributeMap.put(QueueAttributeName.Policy.toString(), policy.toJson());
+
+ // log if permissions are being updated
+ logger.info("updating permissions for queueARN: {}, new policy: {}", queueARN, policy.toJson());
+
+ SetQueueAttributesRequest setQueueAttributesRequest = new SetQueueAttributesRequest(queueUrl, sqsAttributeMap);
+
+ try {
+ sqs.setQueueAttributes(setQueueAttributesRequest);
+ } catch (Exception e) {
+ logger.error("Failed to set permissions on QUEUE ARN=[{}] for TOPIC ARNs=[{}]", queueARN,
+ topicARNs.toString(), e);
+ }
}
}
[4/4] usergrid git commit: Merge commit 'refs/pull/594/head' of
github.com:apache/usergrid
Posted by md...@apache.org.
Merge commit 'refs/pull/594/head' of github.com:apache/usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a88a8752
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a88a8752
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a88a8752
Branch: refs/heads/master
Commit: a88a875271d7789b6366d86d9d45418f70dd3e2b
Parents: c172133 2b35733
Author: Mike Dunker <md...@google.com>
Authored: Wed Dec 13 11:51:39 2017 -0800
Committer: Mike Dunker <md...@google.com>
Committed: Wed Dec 13 11:51:39 2017 -0800
----------------------------------------------------------------------
.../corepersistence/EntityManagerFig.java | 2 +-
.../persistence/queue/LegacyQueueFig.java | 8 +-
.../queue/impl/SNSQueueManagerImpl.java | 4 +-
.../queue/util/AmazonNotificationUtils.java | 135 ++++++++++++++-----
.../collection/CollectionsResourceIT.java | 6 +-
5 files changed, 111 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a88a8752/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a88a8752/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a88a8752/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
[2/4] usergrid git commit: Fix test to use Akka region config instead
of SQS region config
Posted by md...@apache.org.
Fix test to use Akka region config instead of SQS region config
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/71169f89
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/71169f89
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/71169f89
Branch: refs/heads/master
Commit: 71169f89a57a5b6ab5c080022ccde1f80f85fac7
Parents: cda4d20
Author: Mike Dunker <md...@google.com>
Authored: Wed Dec 6 17:04:41 2017 -0800
Committer: Mike Dunker <md...@google.com>
Committed: Wed Dec 6 17:04:41 2017 -0800
----------------------------------------------------------------------
.../rest/applications/collection/CollectionsResourceIT.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/71169f89/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index bf06c21..f172a47 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -1030,7 +1030,7 @@ public class CollectionsResourceIT extends AbstractRestIT {
try {
app().collection( collectionName ).collection( "_settings" )
- .post( new Entity().chainPut(REGION_SETTING, "us-moon-1" ) );
+ .post( new Entity().chainPut(REGION_SETTING, "us-moon" ) );
fail( "post should have failed");
} catch ( BadRequestException expected ) {}
@@ -1038,14 +1038,14 @@ public class CollectionsResourceIT extends AbstractRestIT {
// set collection region with good region
app().collection( collectionName ).collection( "_settings" )
- .post( new Entity().chainPut( REGION_SETTING, "us-east-1" ) );
+ .post( new Entity().chainPut( REGION_SETTING, "us-east" ) );
// get collection settings see that we have a region
collection = app().collection( collectionName ).collection( "_settings" ).get();
settings = (Map<String, Object>)collection.getResponse().getData();
assertNotNull( settings.get( REGION_SETTING ));
- assertEquals( "us-east-1", settings.get( REGION_SETTING ));
+ assertEquals( "us-east", settings.get( REGION_SETTING ));
// unset the collection region