You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2017/10/05 04:49:47 UTC
[01/27] usergrid git commit: increase test cassandra.timeout settings
[Forced Update!]
Repository: usergrid
Updated Branches:
refs/heads/expose-reindex 148337ec8 -> ec8e5c7ca (forced update)
increase test cassandra.timeout settings
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f22c6c4b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f22c6c4b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f22c6c4b
Branch: refs/heads/expose-reindex
Commit: f22c6c4bbe89b6fa94ce223b3381e23ccda4f71c
Parents: a6fee78
Author: Mike Dunker <md...@google.com>
Authored: Wed Aug 16 20:07:45 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Wed Aug 16 20:07:45 2017 -0700
----------------------------------------------------------------------
.../common/src/test/resources/usergrid-UNIT.properties | 2 +-
stack/rest/src/test/resources/usergrid-custom-test.properties | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f22c6c4b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
index 0a43209..13b2937 100644
--- a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
@@ -24,7 +24,7 @@ cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
collections.keyspace=Usergrid_Collections
-cassandra.timeout=10000
+cassandra.timeout=20000
cassandra.embedded=true
collections.keyspace.strategy.options=replication_factor:1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/f22c6c4b/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 615bedd..cbec81f 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -18,6 +18,7 @@
cassandra.startup=external
cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
hystrix.threadpool.graph_user.coreSize=1200
[20/27] usergrid git commit: Change collection clear from PUT to POST.
Posted by mr...@apache.org.
Change collection clear from PUT to POST.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c8605211
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c8605211
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c8605211
Branch: refs/heads/expose-reindex
Commit: c8605211020c53639b5db24075577c272b9d6895
Parents: 3d3ba49
Author: Mike Dunker <md...@google.com>
Authored: Thu Aug 31 17:45:38 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 31 17:45:38 2017 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/rest/applications/CollectionResource.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c8605211/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
index c9174c1..9c09806 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
@@ -196,7 +196,7 @@ public class CollectionResource extends ServiceResource {
}
- @PUT
+ @POST
@Path("{itemName}/_clear")
@Produces({MediaType.APPLICATION_JSON, "application/javascript"})
@RequireApplicationAccess
[02/27] usergrid git commit: change test collection names so no race
condition conflicts
Posted by mr...@apache.org.
change test collection names so no race condition conflicts
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7d9cc895
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7d9cc895
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7d9cc895
Branch: refs/heads/expose-reindex
Commit: 7d9cc89599508d74fdc7e59aec5ab980e55a871a
Parents: f22c6c4
Author: Mike Dunker <md...@google.com>
Authored: Wed Aug 16 22:28:54 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Wed Aug 16 22:28:54 2017 -0700
----------------------------------------------------------------------
.../applications/queries/AndOrQueryTest.java | 111 +++++++--------
.../queries/BadGrammarQueryTest.java | 12 +-
.../rest/applications/queries/OrderByTest.java | 137 ++++++++++---------
.../queries/SelectMappingsQueryTest.java | 53 +++----
4 files changed, 160 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7d9cc895/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
index 4bdd3fc..d49460c 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
@@ -47,20 +47,20 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryAndInclusive() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "apples";
// create our test entities
generateTestEntities(numOfEntities, collectionName);
// Query where madeup = true (the last half) and the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("select * where madeup = true AND ordinal >= " + (numOfEntities - numOfEntities / 4));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
// results should have madeup = true and ordinal 15-19
- assertEquals(numOfEntities / 4, activities.getResponse().getEntityCount());
+ assertEquals(numOfEntities / 4, coll.getResponse().getEntityCount());
// loop though entities that were returned, and test against the ordinals and values we are
// expecting, starting with the last entity and decrementing
int index = 19;
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity activity = coll.next();
// ensure the 'madeup' property is set to true
assertTrue(Boolean.parseBoolean(activity.get("madeup").toString()));
// make sure the correct ordinal properties are returned
@@ -71,15 +71,15 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void someTestProp() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "bananas";
// create our test entities
generateTestEntities(numOfEntities, collectionName);
// Query where madeup = true (the last half) and the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("where sometestprop = 'testprop'");
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
// results should have madeup = true and ordinal 15-19
- assertEquals(10, activities.getResponse().getEntityCount());
+ assertEquals(10, coll.getResponse().getEntityCount());
}
@@ -87,15 +87,15 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void someTestPropPartialContains() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "cantaloupes";
// create our test entities
generateTestEntities(numOfEntities, collectionName);
// Query where madeup = true (the last half) and the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("where sometestprop contains 'test*'");
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
// results should have madeup = true and ordinal 15-19
- assertEquals(10, activities.getResponse().getEntityCount());
+ assertEquals(10, coll.getResponse().getEntityCount());
}
@@ -108,25 +108,25 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryAndExclusive() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "dates";
generateTestEntities(numOfEntities, collectionName);
//Query where madeup = true (the last half) and NOT the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("select * where madeup = true AND NOT ordinal >= " + (numOfEntities - numOfEntities / 4));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//results should have madeup = true and ordinal 10-14
- assertEquals(numOfEntities / 4, activities.getResponse().getEntityCount());
+ assertEquals(numOfEntities / 4, coll.getResponse().getEntityCount());
// loop though entities that were returned, and test against the ordinals and values we are
// expecting, starting with the last expected entity and decrementing
int index = 14;
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//ensure the 'madeup' property is set to true
- assertTrue(Boolean.parseBoolean(activity.get("madeup").toString()));
+ assertTrue(Boolean.parseBoolean(entity.get("madeup").toString()));
//make sure the correct ordinal properties are returned
- assertEquals(index--, Long.parseLong(activity.get("ordinal").toString()));
+ assertEquals(index--, Long.parseLong(entity.get("ordinal").toString()));
}
}
@@ -138,7 +138,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryOrInclusive() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "elderberries";
generateTestEntities(numOfEntities, collectionName);
@@ -146,26 +146,26 @@ public class AndOrQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery("select * where madeup = false OR ordinal >= " + (numOfEntities - numOfEntities / 4))
.setLimit((numOfEntities));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
int index = numOfEntities - 1;
int count = 0;
- int returnSize = activities.getResponse().getEntityCount();
+ int returnSize = coll.getResponse().getEntityCount();
//loop through the returned results
for (int i = 0; i < returnSize; i++, index--) {
count++;
- Entity activity = activities.getResponse().getEntities().get(i);
- logger.info(String.valueOf(activity.get("ordinal")) + " " + String.valueOf(activity.get("madeup")));
+ Entity entity = coll.getResponse().getEntities().get(i);
+ logger.info(String.valueOf(entity.get("ordinal")) + " " + String.valueOf(entity.get("madeup")));
//if the entity is in the first half, the property "madeup" should be false
if (index < numOfEntities / 2) {
- assertFalse(Boolean.parseBoolean(String.valueOf(activity.get("madeup"))));
+ assertFalse(Boolean.parseBoolean(String.valueOf(entity.get("madeup"))));
}
//else if the entity is in the second half, the property "madeup" should be true
else if (index >= (numOfEntities - numOfEntities / 4)) {
- assertTrue(Boolean.parseBoolean(String.valueOf(activity.get("madeup"))));
+ assertTrue(Boolean.parseBoolean(String.valueOf(entity.get("madeup"))));
}
//test to ensure that the ordinal is in the first half (where "madeup = false")
//OR that the ordinal is in the last quarter of the entity list (where "ordinal >= (numOfEntities - numOfEntities / 4))")
- long ordinal = Long.parseLong(String.valueOf(activity.get("ordinal")));
+ long ordinal = Long.parseLong(String.valueOf(entity.get("ordinal")));
assertTrue(ordinal < (numOfEntities / 2) || ordinal >= (numOfEntities - numOfEntities / 4));
}
//results should have madeup = false or ordinal 0-9,15-19
@@ -181,7 +181,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryOrExclusive() throws IOException {
int numOfEntities = 30;
- String collectionName = "activities";
+ String collectionName = "figs";
generateTestEntities(numOfEntities, collectionName);
@@ -189,23 +189,23 @@ public class AndOrQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery("select * where (verb = 'go' OR ordinal >= " + (numOfEntities - numOfEntities / 4) + ") AND NOT (verb = 'go' AND ordinal >= " + (numOfEntities - numOfEntities / 4) + ")")
.setLimit((numOfEntities));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
int index = numOfEntities - 1;
int count = 0;
- int returnSize = activities.getResponse().getEntityCount();
+ int returnSize = coll.getResponse().getEntityCount();
for (int i = 0; i < returnSize; i++, index--) {
count++;
- Entity activity = activities.getResponse().getEntities().get(i);
- long ordinal = Long.parseLong(String.valueOf(activity.get("ordinal")));
- logger.info(ordinal + " " + String.valueOf(activity.get("verb")));
+ Entity entity = coll.getResponse().getEntities().get(i);
+ long ordinal = Long.parseLong(String.valueOf(entity.get("ordinal")));
+ logger.info(ordinal + " " + String.valueOf(entity.get("verb")));
//if the entity is in the first three quarters, the property "verb" should be "go"
if (ordinal < (numOfEntities - numOfEntities / 4)) {
- assertEquals("go", String.valueOf(activity.get("verb")));
+ assertEquals("go", String.valueOf(entity.get("verb")));
}
//if the entity is in the last quarter, the property "verb" should be "stop"
else if (ordinal >= (numOfEntities - numOfEntities / 4)) {
- assertEquals("stop", String.valueOf(activity.get("verb")));
+ assertEquals("stop", String.valueOf(entity.get("verb")));
}
}
//results should be even ordinals in the first 3 quarters and odd ordinals from the last quarter
@@ -225,17 +225,18 @@ public class AndOrQueryTest extends QueryTestBase {
public void queryWithAndPastLimit() throws IOException {
int numValuesTested = 40;
- generateTestEntities(numValuesTested, "activities");
+ String collectionName = "grapes";
+ generateTestEntities(numValuesTested, collectionName);
//3. Query all entities where "madeup = true"
String errorQuery = "select * where madeup = true";
QueryParameters params = new QueryParameters()
.setQuery(errorQuery)
.setLimit(numValuesTested / 2);//4. Limit the query to half of the number of entities
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//5. Ensure the correct entities are returned
- assertEquals(numValuesTested / 2, activities.getResponse().getEntityCount());
- while (activities.hasNext()) {
- assertTrue(Boolean.parseBoolean(activities.next().get("madeup").toString()));
+ assertEquals(numValuesTested / 2, coll.getResponse().getEntityCount());
+ while (coll.hasNext()) {
+ assertTrue(Boolean.parseBoolean(coll.next().get("madeup").toString()));
}
}
@@ -252,16 +253,17 @@ public class AndOrQueryTest extends QueryTestBase {
public void queryNegated() throws IOException {
int numValuesTested = 20;
- generateTestEntities(numValuesTested, "activities");
+ String collectionName = "huckleberries";
+ generateTestEntities(numValuesTested, collectionName);
//1. Query all entities where "NOT verb = 'go'"
String query = "select * where not verb = 'go'";
//2. Limit the query to half of the number of entities
QueryParameters params = new QueryParameters().setQuery(query).setLimit(numValuesTested / 2);
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. Ensure the returned entities have "verb = 'stop'"
- assertEquals(numValuesTested / 2, activities.getResponse().getEntityCount());
- while (activities.hasNext()) {
- assertEquals("stop", activities.next().get("verb").toString());
+ assertEquals(numValuesTested / 2, coll.getResponse().getEntityCount());
+ while (coll.hasNext()) {
+ assertEquals("stop", coll.next().get("verb").toString());
}
@@ -278,15 +280,16 @@ public class AndOrQueryTest extends QueryTestBase {
public void queryReturnCount() throws Exception {
int numValuesTested = 20;
- generateTestEntities(numValuesTested, "activities");
+ String collectionName = "lemons";
+ generateTestEntities(numValuesTested, collectionName);
//1. Query for a subset of the entities
String inCorrectQuery = "select * where ordinal >= " + (numValuesTested / 2) + " order by ordinal asc";
QueryParameters params = new QueryParameters().setQuery(inCorrectQuery).setLimit(numValuesTested / 2);
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//2. Validate that the correct entities are returned
- assertEquals(numValuesTested / 2, activities.getResponse().getEntityCount());
+ assertEquals(numValuesTested / 2, coll.getResponse().getEntityCount());
- List<Entity> entitiesReturned = activities.getResponse().getEntities();
+ List<Entity> entitiesReturned = coll.getResponse().getEntities();
for (int i = 0; i < numValuesTested / 2; i++) {
assertEquals(numValuesTested / 2 + i, Integer.parseInt(entitiesReturned.get(i).get("ordinal").toString()));
}
@@ -303,7 +306,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryCheckAsc() throws Exception {
int numOfEntities = 20;
- String collectionName = "imagination";
+ String collectionName = "melons";
generateTestEntities(numOfEntities, collectionName);
@@ -332,18 +335,18 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryReturnCheck() throws Exception {
int numOfEntities = 20;
- String collectionName = "imagination";
+ String collectionName = "nectarines";
generateTestEntities(numOfEntities, collectionName);
//2. Issue a query
String inquisitiveQuery = String.format("select * where ordinal >= 0 and ordinal <= %d or WhoHelpedYou = 'Ruff' ORDER BY created", numOfEntities);
QueryParameters params = new QueryParameters().setQuery(inquisitiveQuery);
- Collection activities = this.app().collection(collectionName).get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. validate that a full page of (10) entities is returned
- assertEquals(10, activities.getResponse().getEntityCount());
- List<Entity> entitiesReturned = activities.getResponse().getEntities();
+ assertEquals(10, coll.getResponse().getEntityCount());
+ List<Entity> entitiesReturned = coll.getResponse().getEntities();
for (int i = 0; i < 10; i++) {
assertEquals(i, Integer.parseInt(entitiesReturned.get(i).get("ordinal").toString()));
}
@@ -359,7 +362,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryReturnCheckWithShortHand() throws Exception {
int numOfEntities = 10;
- String collectionName = "imagination";
+ String collectionName = "oranges";
generateTestEntities(numOfEntities, collectionName);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7d9cc895/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
index 0692667..950d59d 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
@@ -69,7 +69,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnDoubleQuotes() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "otherthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -95,7 +95,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingQuotes() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "stillotherthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -121,7 +121,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingProperty() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "yetotherthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -147,7 +147,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingPropertyValue() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "thesethings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -173,7 +173,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingOperator() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "thosethings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -200,7 +200,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void limitInQuery() throws IOException {
int numOfEntities =1;
- String collectionName = "things";
+ String collectionName = "whatthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7d9cc895/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
index 6591713..e9bb021 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
@@ -41,21 +41,21 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByLongAsc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "beans";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by ordinal asc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
//results should be ordered by ordinal
int index = 0;
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//make sure the correct ordinal properties are returned
- assertEquals(index++, Long.parseLong(activity.get("ordinal").toString()));
+ assertEquals(index++, Long.parseLong(entity.get("ordinal").toString()));
}
}
@@ -68,23 +68,23 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByLongDesc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "chickpeas";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by ordinal desc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
//Since the sort order is descending, start at the last entity we created
int index = numOfEntities - 1;
//results should be sorted by ordinal
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//make sure the correct ordinal properties are returned
//decrement the index to get the next entity in reverse order
- assertEquals(index--, Long.parseLong(activity.get("ordinal").toString()));
+ assertEquals(index--, Long.parseLong(entity.get("ordinal").toString()));
}
}
@@ -96,25 +96,25 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByBooleanAsc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "lentils";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by madeup asc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted false, then true
//The first half of entities returned should have "madeup = false"
//The second half of entities returned should have "madeup = true"
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
if (index++ < numOfEntities / 2) {
- assertEquals("false", activity.get("madeup").toString());
+ assertEquals("false", entity.get("madeup").toString());
} else {
- assertEquals("true", activity.get("madeup").toString());
+ assertEquals("true", entity.get("madeup").toString());
}
}
}
@@ -127,26 +127,26 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByBooleanDesc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "peas";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by madeup desc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted true, then false
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//make sure the booleans are ordered correctly
//The first half of entities returned should have "madeup = true"
//The second half of entities returned should have "madeup = false"
if (index++ < numOfEntities / 2) {
- assertEquals("true", activity.get("madeup").toString());
+ assertEquals("true", entity.get("madeup").toString());
} else {
- assertEquals("false", activity.get("madeup").toString());
+ assertEquals("false", entity.get("madeup").toString());
}
}
}
@@ -159,25 +159,25 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByStringAsc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "carrots";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
//Sort by the "verb" property to test alphabetical sorting of string properties
QueryParameters params = new QueryParameters()
.setQuery("select * order by verb asc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted "go", then "stop"
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//The first half of entities returned should have "verb = 'go'"
//The second half of entities returned should have "verb = 'stop'"
if (index++ < numOfEntities / 2) {
- assertEquals("go", activity.get("verb").toString());
+ assertEquals("go", entity.get("verb").toString());
} else {
- assertEquals("stop", activity.get("verb").toString());
+ assertEquals("stop", entity.get("verb").toString());
}
}
}
@@ -190,7 +190,7 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByStringDesc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "celery";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -198,18 +198,18 @@ public class OrderByTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery("select * order by verb desc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted "stop", then "go"
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//The first half of entities returned should have "verb = 'stop'"
//The second half of entities returned should have "verb = 'go'"
if (index++ < numOfEntities / 2) {
- assertEquals("stop", activity.get("verb").toString());
+ assertEquals("stop", entity.get("verb").toString());
} else {
- assertEquals("go", activity.get("verb").toString());
+ assertEquals("go", entity.get("verb").toString());
}
}
@@ -229,6 +229,7 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByShouldNotAffectResults() throws IOException {
+ String collectionName = "mushrooms";
long created = 0;
Entity actor = new Entity();
actor.put("displayName", "Erin");
@@ -239,10 +240,10 @@ public class OrderByTest extends QueryTestBase {
//1. Insert entities
for (int i = 0; i < 20; i++) {
props.put("ordinal", i);
- Entity activity = this.app().collection("activity").post(props);
- logger.info("Created", activity.get("created").toString());
+ Entity entity = this.app().collection(collectionName).post(props);
+ logger.info("Created", entity.get("created").toString());
if (i == 5) {
- created = Long.parseLong(activity.get("created").toString());
+ created = Long.parseLong(entity.get("created").toString());
}
}
@@ -250,18 +251,18 @@ public class OrderByTest extends QueryTestBase {
//2. Query without 'order by'
String query = "select * where created > " + created;
QueryParameters params = new QueryParameters().setQuery(query);
- Collection activitiesWithoutOrderBy = this.app().collection("activities").get(params);
- assertEquals(10, activitiesWithoutOrderBy.getResponse().getEntityCount());
+ Collection entitiesWithoutOrderBy = this.app().collection(collectionName).get(params);
+ assertEquals(10, entitiesWithoutOrderBy.getResponse().getEntityCount());
//3. Query with 'order by'
query = query + " order by created desc";
params.setQuery(query);
- Collection activitiesWithOrderBy = this.app().collection("activities").get(params);
+ Collection activitiesWithOrderBy = this.app().collection(collectionName).get(params);
assertEquals(10, activitiesWithOrderBy.getResponse().getEntityCount());
//4. Ensure the same entities are returned
- while (activitiesWithoutOrderBy.hasNext() && activitiesWithOrderBy.hasNext()) {
- Entity activityWithoutOrderBy = activitiesWithoutOrderBy.next();
- Entity activityWithOrderBy = activitiesWithOrderBy.next();
- assertEquals(activityWithoutOrderBy.get("uuid").toString(), activityWithOrderBy.get("uuid").toString());
+ while (entitiesWithoutOrderBy.hasNext() && activitiesWithOrderBy.hasNext()) {
+ Entity entityWithoutOrderBy = entitiesWithoutOrderBy.next();
+ Entity entityWithOrderBy = activitiesWithOrderBy.next();
+ assertEquals(entityWithoutOrderBy.get("uuid").toString(), entityWithOrderBy.get("uuid").toString());
}
}
@@ -277,6 +278,7 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByComesBeforeLimitResult() throws IOException {
+ String collectionName = "onions";
Entity actor = new Entity();
actor.put("displayName", "Erin");
Entity props = new Entity();
@@ -286,23 +288,23 @@ public class OrderByTest extends QueryTestBase {
//1. Insert entities
for (int i = 0; i < 20; i++) {
props.put("ordinal", i);
- this.app().collection("activity").post(props);
+ this.app().collection(collectionName).post(props);
}
waitForQueueDrainAndRefreshIndex();
//2. Query a subset of the entities, specifying order and limit
String query = "select * where created > " + 1 + " order by created desc";
QueryParameters params = new QueryParameters().setQuery(query).setLimit(5);
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. Ensure the correct number of results are returned
- assertEquals(5, activities.getResponse().getEntityCount());
+ assertEquals(5, coll.getResponse().getEntityCount());
//2. Query a subset of the entities, specifying order and limit
query = " where created > " + 1 + " order by created desc";
params = new QueryParameters().setQuery(query).setLimit(5);
- activities = this.app().collection("activities").get(params);
+ coll = this.app().collection(collectionName).get(params);
//3. Ensure the correct number of results are returned
- assertEquals(5, activities.getResponse().getEntityCount());
+ assertEquals(5, coll.getResponse().getEntityCount());
}
/**
@@ -316,8 +318,9 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByReturnCorrectResults() throws IOException {
+ String collectionName = "peppers";
int size = 20;
- Entity[] activities = new Entity[size];
+ Entity[] entities = new Entity[size];
Entity actor = new Entity();
actor.put("displayName", "Erin");
@@ -328,34 +331,34 @@ public class OrderByTest extends QueryTestBase {
//1. Insert a number of entities and add them to an array
for (int i = 0; i < size; i++) {
props.put("ordinal", i);
- Entity e = this.app().collection("activity").post(props);
- activities[i] = e;
+ Entity e = this.app().collection(collectionName).post(props);
+ entities[i] = e;
logger.info(String.valueOf(e.get("uuid").toString()));
- logger.info(String.valueOf(Long.parseLong(activities[0].get("created").toString())));
+ logger.info(String.valueOf(Long.parseLong(entities[0].get("created").toString())));
}
waitForQueueDrainAndRefreshIndex(750);
- ArrayUtils.reverse(activities);
- long lastCreated = Long.parseLong(activities[0].get("created").toString());
+ ArrayUtils.reverse(entities);
+ long lastCreated = Long.parseLong(entities[0].get("created").toString());
//2. Query for the entities in descending order
String errorQuery = String.format("select * where created <= %d order by created desc", lastCreated);
int index = 0;
QueryParameters params = new QueryParameters().setQuery(errorQuery);
- Collection activitiesResponse = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. Validate that the order is correct
do {
- int returnSize = activitiesResponse.getResponse().getEntityCount();
+ int returnSize = coll.getResponse().getEntityCount();
//loop through the current page of results
for (int i = 0; i < returnSize; i++, index++) {
- assertEquals( ( activities[index] ).get( "uuid" ).toString(),
- activitiesResponse.getResponse().getEntities().get(i).get("uuid").toString());
+ assertEquals( ( entities[index] ).get( "uuid" ).toString(),
+ coll.getResponse().getEntities().get(i).get("uuid").toString());
}
//grab the next page of results
- activitiesResponse = this.app().collection("activities").getNextPage(activitiesResponse, params, true);
+ coll = this.app().collection(collectionName).getNextPage(coll, params, true);
}
- while (activitiesResponse.getCursor() != null);
+ while (coll.getCursor() != null);
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7d9cc895/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
index acf51c1..cb103cf 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
@@ -41,14 +41,15 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testNestedSelectFieldNames() throws Exception {
- generateTestEntities(20, "things");
+ String collectionName = "basketballs";
+ generateTestEntities(20, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select actor.displayName,sometestprop where sometestprop = 'testprop'");
- Collection things = this.app().collection("things").get(params);
- assertEquals( 10, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals( 10, coll.getNumOfEntities() );
- Iterator<Entity> iter = things.iterator();
+ Iterator<Entity> iter = coll.iterator();
while ( iter.hasNext() ) {
Entity entity = iter.next();
@@ -73,7 +74,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testMixedCaseDupField() throws Exception {
- String collectionName = "things";
+ String collectionName = "baseballs";
String value = RandomStringUtils.randomAlphabetic( 20 );
String otherValue = RandomStringUtils.randomAlphabetic( 20 );
@@ -89,19 +90,19 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + otherValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
params = new QueryParameters()
.setQuery( "select * where TESTPROP='" + otherValue + "'" );
- things = app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
@Test
public void testStringWithSingleQuote() throws Exception {
- String collectionName = "things";
+ String collectionName = "footballs";
String value = "test'value";
String escapedValue = "test\\'value";
@@ -116,14 +117,14 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testprop='" + escapedValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
+ Collection things = this.app().collection(collectionName).get( params );
assertEquals( 1, things.getNumOfEntities() );
}
@Test
public void testStringWithPlus() throws Exception {
- String collectionName = "things";
+ String collectionName = "volleyballs";
String value = "ed+test@usergrid.com";
// create entity with value containing a plus symbol
@@ -135,15 +136,15 @@ public class SelectMappingsQueryTest extends QueryTestBase {
// now query this without encoding the plus symbol
QueryParameters params = new QueryParameters()
.setQuery( "select * where testprop='" + value + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
// again query with the plus symbol url encoded
String escapedValue = "ed%2Btest@usergrid.com";
params = new QueryParameters()
.setQuery( "select * where testprop='" + escapedValue + "'" );
- things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
@@ -154,7 +155,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testFieldOverride1() throws Exception {
- String collectionName = "things";
+ String collectionName = "pickleballs";
// create entity with testProp=value
String value = RandomStringUtils.randomAlphabetic( 20 );
@@ -172,13 +173,13 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + newValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
params = new QueryParameters()
.setQuery( "select * where TESTPROP='" + newValue + "'" );
- things = app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
/**
@@ -187,7 +188,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testFieldOverride2() throws Exception {
- String collectionName = "things";
+ String collectionName = "tennisballs";
// create entity with TESTPROP=value
String value = RandomStringUtils.randomAlphabetic( 20 );
@@ -205,13 +206,13 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + newValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
params = new QueryParameters()
.setQuery( "select * where TESTPROP='" + newValue + "'" );
- things = app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
}
[16/27] usergrid git commit: 1. Add missing start for delete queue
workers. 2. Add dropping of application local keyspace for testing. 3.
Testing changes.
Posted by mr...@apache.org.
1. Add missing start for delete queue workers.
2. Add dropping of application local keyspace for testing.
3. Testing changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3f7afcd8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3f7afcd8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3f7afcd8
Branch: refs/heads/expose-reindex
Commit: 3f7afcd8836ed9b589b69523d8bcc507a739d912
Parents: c201f1f
Author: Mike Dunker <md...@google.com>
Authored: Mon Aug 21 17:01:48 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Aug 21 17:01:48 2017 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 3 +--
.../asyncevents/AsyncEventServiceImpl.java | 4 +++
.../index/CollectionSettingsImpl.java | 1 -
.../usergrid/persistence/EntityManager.java | 2 +-
.../persistence/entities/Application.java | 26 ++++++++++++++++++++
.../usergrid/persistence/CoreSchemaManager.java | 16 ++++++++----
.../usergrid/persistence/RebuildIndexTest.java | 24 +++++++++++++-----
.../resources/usergrid-custom-test.properties | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
.../AbstractServiceNotificationIT.java | 4 +--
.../gcm/NotificationsServiceIT.java | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
12 files changed, 67 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index cdb4fc7..68c4ef0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2471,10 +2471,9 @@ public class CpEntityManager implements EntityManager {
final Entity entity;
- //this is the fall back, why isn't this writt
if ( entityType == null ) {
return null;
-// throw new EntityNotFoundException( String.format( "Counld not find type for uuid {}", uuid ) );
+// throw new EntityNotFoundException( String.format( "Could not find type for uuid {}", uuid ) );
}
entity = get( new SimpleEntityRef( entityType, uuid ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 79a80c0..428772f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -875,6 +875,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
startWorker(AsyncEventQueueType.UTILITY);
}
+ for (int i = 0; i < deleteCount; i++) {
+ startWorker(AsyncEventQueueType.DELETE);
+ }
+
if( indexQueue instanceof SNSQueueManagerImpl) {
logger.info("Queue manager implementation supports dead letters, start dead letter queue workers.");
for (int i = 0; i < indexDeadCount; i++) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
index 921777a..76f0b4b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.corepersistence.index;
import com.google.common.base.Optional;
import com.google.inject.Inject;
-import com.google.inject.Singleton;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.utils.JsonUtils;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index ae4623d..c9752c3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -743,7 +743,7 @@ public interface EntityManager {
/**
* Add a new index to the application for scale
- * @param suffix unique indentifier for additional index
+ * @param newIndexName unique identifier for additional index
* @param shards number of shards
* @param replicas number of replicas
* @param writeConsistency only "one, quorum, or all"
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
index 790b4d9..0a4360f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
@@ -48,6 +48,32 @@ public class Application extends TypedEntity implements Serializable {
public static final String COLLECTION_ACTIVITIES = "activities";
+ public static final String COLLECTION_EVENTS = "events";
+
+ public static final String COLLECTION_FOLDERS = "folders";
+
+ public static final String COLLECTION_DEVICES = "devices";
+
+ public static final String COLLECTION_NOTIFICATIONS = "notifications";
+
+ public static final String COLLECTION_ROLES = "roles";
+
+ public static boolean isCustomCollectionName(String collectionName) {
+ switch (collectionName.toLowerCase()) {
+ case COLLECTION_USERS:
+ case COLLECTION_GROUPS:
+ case COLLECTION_ASSETS:
+ case COLLECTION_ACTIVITIES:
+ case COLLECTION_EVENTS:
+ case COLLECTION_FOLDERS:
+ case COLLECTION_DEVICES:
+ case COLLECTION_NOTIFICATIONS:
+ case COLLECTION_ROLES:
+ return false;
+ }
+ return true;
+ }
+
@EntityProperty(indexed = true, fulltextIndexed = false, required = true, mutable = false, aliasProperty = true,
basic = true)
protected String name;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
index c064751..1d51c25 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.persistence;
import org.apache.usergrid.locking.LockManager;
+import org.apache.usergrid.persistence.core.CassandraConfig;
import org.apache.usergrid.persistence.core.CassandraFig;
import org.apache.usergrid.persistence.core.datastax.CQLUtils;
import org.apache.usergrid.persistence.core.datastax.DataStaxCluster;
@@ -37,14 +38,14 @@ public class CoreSchemaManager implements SchemaManager {
private static final Logger logger = LoggerFactory.getLogger( CoreSchemaManager.class );
private final Setup setup;
- private final CassandraFig cassandraFig;
+ private final CassandraConfig cassandraConfig;
private final LockManager lockManager;
private final DataStaxCluster dataStaxCluster;
public CoreSchemaManager( final Setup setup, Injector injector ) {
this.setup = setup;
- this.cassandraFig = injector.getInstance( CassandraFig.class );
+ this.cassandraConfig = injector.getInstance( CassandraConfig.class );
this.lockManager = injector.getInstance( LockManager.class );
this.dataStaxCluster = injector.getInstance( DataStaxCluster.class );
}
@@ -80,16 +81,21 @@ public class CoreSchemaManager implements SchemaManager {
@Override
public void destroy() {
- logger.info( "dropping keyspaces" );
try {
+ logger.info( "dropping application keyspace" );
dataStaxCluster.getClusterSession()
- .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraFig.getApplicationKeyspace()));
+ .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationKeyspace()));
+ dataStaxCluster.waitForSchemaAgreement();
+
+ logger.info( "dropping application local keyspace" );
+ dataStaxCluster.getClusterSession()
+ .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationLocalKeyspace()));
dataStaxCluster.waitForSchemaAgreement();
dataStaxCluster.getClusterSession().close(); // close session so it's meta will get refreshed
}
catch ( Exception e ) {
- logger.error("Error dropping application keyspace: {} error: {}", cassandraFig.getApplicationKeyspace(), e);
+ logger.error("Error dropping application keyspaces: {} error: {}", cassandraConfig.getApplicationKeyspace(), e);
}
logger.info( "keyspaces dropped" );
logger.info( "dropping indices" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index d2bff37..a1ff84b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -74,7 +74,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
}
- @Test( timeout = 120000 )
+ @Test( timeout = 240000 )
public void rebuildOneCollectionIndex() throws Exception {
logger.info( "Started rebuildOneCollectionIndex()" );
@@ -163,7 +163,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
waitForRebuild( status, reIndexService );
- app.waitForQueueDrainAndRefreshIndex(5000);
+ app.waitForQueueDrainAndRefreshIndex(15000);
// ----------------- test that we can read the catherder collection and not the catshepard
@@ -278,7 +278,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
assertNotNull( status.getJobId(), "JobId is present" );
- logger.info( "Rebuilt index" );
+ logger.info( "Rebuilt index, jobID={}", status.getJobId());
waitForRebuild( status, reIndexService );
@@ -511,14 +511,26 @@ public class RebuildIndexTest extends AbstractCoreIT {
* Wait for the rebuild to occur
*/
private void waitForRebuild( final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService )
- throws InterruptedException {
+ throws InterruptedException, IllegalArgumentException {
+ if (status != null) {
+ logger.info("waitForRebuild: jobID={}", status.getJobId());
+ } else {
+ logger.info("waitForRebuild: error, status = null");
+ throw new IllegalArgumentException("reindexStatus = null");
+ }
while ( true ) {
try {
final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus( status.getJobId() );
- if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
- break;
+ if (updatedStatus == null) {
+ logger.info("waitForRebuild: updated status is null");
+ } else {
+ logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed());
+
+ if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
+ break;
+ }
}
}
catch ( IllegalArgumentException iae ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index df253f0..24f3046 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -29,7 +29,7 @@ elasticsearch.management_number_replicas=0
elasticsearch.managment_index=usergrid_core_management
#cassandra.keyspace.application=core_tests_schema
elasticsearch.queue_impl.resolution=true
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
elasticsearch.buffer_timeout=1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 4b76ca4..ee7b039 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -68,7 +68,7 @@ usergrid.cluster.port=2555
collection.uniquevalues.actors=300
collection.uniquevalues.authoritative.region=us-east
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
# Queueing Test Settings
# Reduce the long polling time for the tests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
index c035192..9b3dfcd 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
@@ -49,9 +49,9 @@ public abstract class AbstractServiceNotificationIT extends AbstractServiceIT {
protected Notification notificationWaitForComplete(Notification notification)
throws Exception {
- long timeout = System.currentTimeMillis() + 60000;
+ long timeout = System.currentTimeMillis() + 120000;
while (System.currentTimeMillis() < timeout) {
- app.waitForQueueDrainAndRefreshIndex(200);
+ app.waitForQueueDrainAndRefreshIndex(1000);
notification = app.getEntityManager().get(notification.getUuid(), Notification.class);
if (notification.getFinished() != null) {
return notification;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 8360009..ecc5443 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -632,7 +632,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
// receipts are created and queried, wait a bit longer for this to happen as indexing
- app.waitForQueueDrainAndRefreshIndex(500);
+ app.waitForQueueDrainAndRefreshIndex(5000);
// get the receipts entity IDs
List<EntityRef> receipts = getNotificationReceipts(notification);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties
index bcc8b8e..4178a07 100644
--- a/stack/services/src/test/resources/usergrid-custom-test.properties
+++ b/stack/services/src/test/resources/usergrid-custom-test.properties
@@ -35,7 +35,7 @@ elasticsearch.managment_index=usergrid_services_management
elasticsearch.buffer_timeout=1
elasticsearch.queue_impl.resolution=true
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
# Queueing Test Settings
queue.long.polling.time.millis=50
[21/27] usergrid git commit: Handle IndexMissingException when
fetching the index size for an application.
Posted by mr...@apache.org.
Handle IndexMissingException when fetching the index size for an application.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/41702956
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/41702956
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/41702956
Branch: refs/heads/expose-reindex
Commit: 41702956d5aedac5a8e8d44f8f3df7af5ec365b9
Parents: c860521
Author: Michael Russo <ru...@google.com>
Authored: Tue Sep 12 18:42:43 2017 -0700
Committer: Michael Russo <ru...@google.com>
Committed: Tue Sep 12 18:42:43 2017 -0700
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 29 ++++++++++++++------
1 file changed, 20 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/41702956/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index f4fae2b..3d2f576 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -70,6 +70,7 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.indices.IndexAlreadyExistsException;
+import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
@@ -804,15 +805,25 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
private long getIndexSize(){
- final IndicesStatsResponse statsResponse = esProvider.getClient()
- .admin()
- .indices()
- .prepareStats(indexLocationStrategy.getIndexInitialName())
- .all()
- .execute()
- .actionGet();
- final CommonStats indexStats = statsResponse.getIndex(indexLocationStrategy.getIndexInitialName()).getTotal();
- return indexStats.getStore().getSizeInBytes();
+ long indexSize = 0L;
+ final String indexName = indexLocationStrategy.getIndexInitialName();
+ try {
+ final IndicesStatsResponse statsResponse = esProvider.getClient()
+ .admin()
+ .indices()
+ .prepareStats(indexName)
+ .all()
+ .execute()
+ .actionGet();
+ final CommonStats indexStats = statsResponse.getIndex(indexName).getTotal();
+ indexSize = indexStats.getStore().getSizeInBytes();
+ } catch (IndexMissingException e) {
+ // if for some reason the index size does not exist,
+ // log an error and we can assume size is 0 as it doesn't exist
+ logger.error("Unable to get size for index {} due to IndexMissingException for app {}",
+ indexName, indexLocationStrategy.getApplicationScope().getApplication().getUuid());
+ }
+ return indexSize;
}
@Override
[12/27] usergrid git commit: test changes and additional logging
Posted by mr...@apache.org.
test changes and additional logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/39ec4f2b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/39ec4f2b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/39ec4f2b
Branch: refs/heads/expose-reindex
Commit: 39ec4f2bb895e180f493349fb14e62ac9178e5ee
Parents: 400365c
Author: Mike Dunker <md...@google.com>
Authored: Fri Aug 18 09:46:50 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Fri Aug 18 09:46:50 2017 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../asyncevents/AsyncEventServiceImpl.java | 2 ++
.../apache/usergrid/persistence/RebuildIndexTest.java | 12 ++++++------
.../queues/impl/QueueSerializationImpl.java | 2 +-
.../apache/usergrid/services/ActivitiesServiceIT.java | 9 ++++++---
.../apache/usergrid/services/ServiceInvocationIT.java | 3 +++
6 files changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index cec7258..bad5b2c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -487,7 +487,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// evict app Id from cache
applicationIdCache.evictAppId(appName);
- logger.info("Initialized application {}", appName);
+ logger.info("Initialized application {}, uuid {}", appName, appInfo.getUuid().toString());
return appInfo;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 530cf7d..9501ad3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -881,6 +881,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final int utilityCount = indexProcessorFig.getWorkerCountUtility();
final int indexDeadCount = indexProcessorFig.getWorkerCountDeadLetter();
final int utilityDeadCount = indexProcessorFig.getWorkerCountUtilityDeadLetter();
+ logger.info("Starting queue workers for indexing: index={} indexDLQ={} utility={} utilityDLQ={}", indexCount, indexDeadCount,
+ utilityCount, utilityDeadCount);
for (int i = 0; i < indexCount; i++) {
startWorker(QUEUE_NAME);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index a7759de..d2bff37 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -77,7 +77,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
@Test( timeout = 120000 )
public void rebuildOneCollectionIndex() throws Exception {
- logger.info( "Started rebuildIndex()" );
+ logger.info( "Started rebuildOneCollectionIndex()" );
String rand = RandomStringUtils.randomAlphanumeric( 5 );
final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
@@ -172,7 +172,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
}
- @Test( timeout = 120000 )
+ @Test( timeout = 240000 )
public void rebuildIndex() throws Exception {
logger.info( "Started rebuildIndex()" );
@@ -234,7 +234,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
}
logger.info( "Created {} entities", ENTITIES_TO_INDEX );
- app.waitForQueueDrainAndRefreshIndex(15000);
+ app.waitForQueueDrainAndRefreshIndex(30000);
// ----------------- test that we can read them, should work fine
@@ -301,7 +301,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
@Test( timeout = 120000 )
public void rebuildIndexGeo() throws Exception {
- logger.info( "Started rebuildIndex()" );
+ logger.info( "Started rebuildIndexGeo()" );
String rand = RandomStringUtils.randomAlphanumeric( 5 );
final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
@@ -414,7 +414,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
@Test( timeout = 120000 )
public void rebuildUpdatedSince() throws Exception {
- logger.info( "Started rebuildIndex()" );
+ logger.info( "Started rebuildUpdatedSince()" );
String rand = RandomStringUtils.randomAlphanumeric( 5 );
final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
@@ -436,7 +436,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
final Entity secondEntity = em.create( "thing", entityData);
- app.waitForQueueDrainAndRefreshIndex(5000);
+ app.waitForQueueDrainAndRefreshIndex(15000);
// ----------------- test that we can read them, should work fine
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java
index d3a46aa..2ed37e0 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/qakka/serialization/queues/impl/QueueSerializationImpl.java
@@ -163,7 +163,7 @@ public class QueueSerializationImpl implements QueueSerialization {
@Override
public Collection<TableDefinition> getTables() {
return Collections.singletonList(
- new TableDefinitionStringImpl( cassandraConfig.getApplicationKeyspace(), "queues", CQL ) );
+ new TableDefinitionStringImpl( cassandraConfig.getApplicationKeyspace(), TABLE_QUEUES, CQL ) );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java
index c4762f2..c8a446a 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ActivitiesServiceIT.java
@@ -93,11 +93,14 @@ public class ActivitiesServiceIT extends AbstractServiceIT {
app.testRequest( ServiceAction.GET, 4, null, "users", userC.getUuid(), "feed" );
- app.testRequest( ServiceAction.GET, 2, null, "users", userC.getUuid(), "feed",
- Query.fromQL( "select * where content contains 'cookie'" ) );
+ // time for indexing
+ Thread.sleep(10000);
app.testRequest( ServiceAction.GET, 1, "users", userC.getUuid(), "feed",
- Query.fromQL( "select * where verb='post' and content contains 'cookie'" ) );
+ Query.fromQL( "select * where verb='post' and content contains 'cookie'" ) );
+
+ app.testRequest( ServiceAction.GET, 2, null, "users", userC.getUuid(), "feed",
+ Query.fromQL( "select * where content contains 'cookie'" ) );
app.put( "username", "finn" );
app.put( "email", "finn@ooo.com" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/39ec4f2b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
index 81dced1..434fd80 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceInvocationIT.java
@@ -102,6 +102,9 @@ public class ServiceInvocationIT extends AbstractServiceIT {
app.testRequest( ServiceAction.GET, 2, "users", "edanuff", "likes", "restaurants" );
+ // time for indexing
+ Thread.sleep(10000);
+
app.testRequest( ServiceAction.GET, 1, "users", "edanuff", "likes", "restaurants",
Query.fromQL( "select * where name='Brickhouse'" ) );
[04/27] usergrid git commit: Add new parameter that can be used in
the JSP viewables.
Posted by mr...@apache.org.
Add new parameter that can be used in the JSP viewables.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/068263ed
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/068263ed
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/068263ed
Branch: refs/heads/expose-reindex
Commit: 068263ed437883edeba1d23a024f62bb6e7b0700
Parents: a6fee78
Author: Michael Russo <ru...@google.com>
Authored: Wed May 24 21:45:04 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 09:41:40 2017 -0700
----------------------------------------------------------------------
stack/config/src/main/resources/usergrid-default.properties | 1 +
.../apache/usergrid/rest/management/users/UserResource.java | 7 +++++++
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/068263ed/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index d505a96..77b047e 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -786,6 +786,7 @@ usergrid.api.url.base=http://localhost:8080/ROOT
# instead, use character class ([.] instead of backslash-period)
usergrid.org.config.property.regex=usergrid[.]view[.].*
+usergrid.viewable.loginEndpoint=http://localhost:8080
########################### Usergrid Email Templates ########################
http://git-wip-us.apache.org/repos/asf/usergrid/blob/068263ed/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index 4cbe9b2..b747aa4 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -66,6 +66,9 @@ public class UserResource extends AbstractContextResource {
String token = null;
+ String loginEndpoint;
+
+
public UserResource() {
}
@@ -296,6 +299,7 @@ public class UserResource extends AbstractContextResource {
if ( ( password1 != null ) && password1.equals( password2 ) ) {
management.setAdminUserPassword( user.getUuid(), password1 );
management.revokeAccessTokenForAdminUser( user.getUuid(), token );
+ loginEndpoint = properties.getProperty("usergrid.viewable.loginEndpoint");
return handleViewable( "resetpw_set_success", this, organizationId );
}
else {
@@ -342,6 +346,9 @@ public class UserResource extends AbstractContextResource {
return errorMsg;
}
+ public String getLoginEndpoint() {
+ return loginEndpoint;
+ }
public String getToken() {
return token;
[09/27] usergrid git commit: change test collection names so no race
condition conflicts
Posted by mr...@apache.org.
change test collection names so no race condition conflicts
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/d06dbc95
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/d06dbc95
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/d06dbc95
Branch: refs/heads/expose-reindex
Commit: d06dbc95a5e41a9cebb20f5859860dd74b6ae9ac
Parents: 4b916c4
Author: Mike Dunker <md...@google.com>
Authored: Wed Aug 16 22:28:54 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 11:59:47 2017 -0700
----------------------------------------------------------------------
.../applications/queries/AndOrQueryTest.java | 111 +++++++--------
.../queries/BadGrammarQueryTest.java | 12 +-
.../rest/applications/queries/OrderByTest.java | 137 ++++++++++---------
.../queries/SelectMappingsQueryTest.java | 53 +++----
4 files changed, 160 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d06dbc95/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
index 4bdd3fc..d49460c 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/AndOrQueryTest.java
@@ -47,20 +47,20 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryAndInclusive() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "apples";
// create our test entities
generateTestEntities(numOfEntities, collectionName);
// Query where madeup = true (the last half) and the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("select * where madeup = true AND ordinal >= " + (numOfEntities - numOfEntities / 4));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
// results should have madeup = true and ordinal 15-19
- assertEquals(numOfEntities / 4, activities.getResponse().getEntityCount());
+ assertEquals(numOfEntities / 4, coll.getResponse().getEntityCount());
// loop though entities that were returned, and test against the ordinals and values we are
// expecting, starting with the last entity and decrementing
int index = 19;
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity activity = coll.next();
// ensure the 'madeup' property is set to true
assertTrue(Boolean.parseBoolean(activity.get("madeup").toString()));
// make sure the correct ordinal properties are returned
@@ -71,15 +71,15 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void someTestProp() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "bananas";
// create our test entities
generateTestEntities(numOfEntities, collectionName);
// Query where madeup = true (the last half) and the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("where sometestprop = 'testprop'");
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
// results should have madeup = true and ordinal 15-19
- assertEquals(10, activities.getResponse().getEntityCount());
+ assertEquals(10, coll.getResponse().getEntityCount());
}
@@ -87,15 +87,15 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void someTestPropPartialContains() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "cantaloupes";
// create our test entities
generateTestEntities(numOfEntities, collectionName);
// Query where madeup = true (the last half) and the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("where sometestprop contains 'test*'");
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
// results should have madeup = true and ordinal 15-19
- assertEquals(10, activities.getResponse().getEntityCount());
+ assertEquals(10, coll.getResponse().getEntityCount());
}
@@ -108,25 +108,25 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryAndExclusive() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "dates";
generateTestEntities(numOfEntities, collectionName);
//Query where madeup = true (the last half) and NOT the last quarter of entries
QueryParameters params = new QueryParameters()
.setQuery("select * where madeup = true AND NOT ordinal >= " + (numOfEntities - numOfEntities / 4));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//results should have madeup = true and ordinal 10-14
- assertEquals(numOfEntities / 4, activities.getResponse().getEntityCount());
+ assertEquals(numOfEntities / 4, coll.getResponse().getEntityCount());
// loop though entities that were returned, and test against the ordinals and values we are
// expecting, starting with the last expected entity and decrementing
int index = 14;
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//ensure the 'madeup' property is set to true
- assertTrue(Boolean.parseBoolean(activity.get("madeup").toString()));
+ assertTrue(Boolean.parseBoolean(entity.get("madeup").toString()));
//make sure the correct ordinal properties are returned
- assertEquals(index--, Long.parseLong(activity.get("ordinal").toString()));
+ assertEquals(index--, Long.parseLong(entity.get("ordinal").toString()));
}
}
@@ -138,7 +138,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryOrInclusive() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "elderberries";
generateTestEntities(numOfEntities, collectionName);
@@ -146,26 +146,26 @@ public class AndOrQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery("select * where madeup = false OR ordinal >= " + (numOfEntities - numOfEntities / 4))
.setLimit((numOfEntities));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
int index = numOfEntities - 1;
int count = 0;
- int returnSize = activities.getResponse().getEntityCount();
+ int returnSize = coll.getResponse().getEntityCount();
//loop through the returned results
for (int i = 0; i < returnSize; i++, index--) {
count++;
- Entity activity = activities.getResponse().getEntities().get(i);
- logger.info(String.valueOf(activity.get("ordinal")) + " " + String.valueOf(activity.get("madeup")));
+ Entity entity = coll.getResponse().getEntities().get(i);
+ logger.info(String.valueOf(entity.get("ordinal")) + " " + String.valueOf(entity.get("madeup")));
//if the entity is in the first half, the property "madeup" should be false
if (index < numOfEntities / 2) {
- assertFalse(Boolean.parseBoolean(String.valueOf(activity.get("madeup"))));
+ assertFalse(Boolean.parseBoolean(String.valueOf(entity.get("madeup"))));
}
//else if the entity is in the second half, the property "madeup" should be true
else if (index >= (numOfEntities - numOfEntities / 4)) {
- assertTrue(Boolean.parseBoolean(String.valueOf(activity.get("madeup"))));
+ assertTrue(Boolean.parseBoolean(String.valueOf(entity.get("madeup"))));
}
//test to ensure that the ordinal is in the first half (where "madeup = false")
//OR that the ordinal is in the last quarter of the entity list (where "ordinal >= (numOfEntities - numOfEntities / 4))")
- long ordinal = Long.parseLong(String.valueOf(activity.get("ordinal")));
+ long ordinal = Long.parseLong(String.valueOf(entity.get("ordinal")));
assertTrue(ordinal < (numOfEntities / 2) || ordinal >= (numOfEntities - numOfEntities / 4));
}
//results should have madeup = false or ordinal 0-9,15-19
@@ -181,7 +181,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryOrExclusive() throws IOException {
int numOfEntities = 30;
- String collectionName = "activities";
+ String collectionName = "figs";
generateTestEntities(numOfEntities, collectionName);
@@ -189,23 +189,23 @@ public class AndOrQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery("select * where (verb = 'go' OR ordinal >= " + (numOfEntities - numOfEntities / 4) + ") AND NOT (verb = 'go' AND ordinal >= " + (numOfEntities - numOfEntities / 4) + ")")
.setLimit((numOfEntities));
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
int index = numOfEntities - 1;
int count = 0;
- int returnSize = activities.getResponse().getEntityCount();
+ int returnSize = coll.getResponse().getEntityCount();
for (int i = 0; i < returnSize; i++, index--) {
count++;
- Entity activity = activities.getResponse().getEntities().get(i);
- long ordinal = Long.parseLong(String.valueOf(activity.get("ordinal")));
- logger.info(ordinal + " " + String.valueOf(activity.get("verb")));
+ Entity entity = coll.getResponse().getEntities().get(i);
+ long ordinal = Long.parseLong(String.valueOf(entity.get("ordinal")));
+ logger.info(ordinal + " " + String.valueOf(entity.get("verb")));
//if the entity is in the first three quarters, the property "verb" should be "go"
if (ordinal < (numOfEntities - numOfEntities / 4)) {
- assertEquals("go", String.valueOf(activity.get("verb")));
+ assertEquals("go", String.valueOf(entity.get("verb")));
}
//if the entity is in the last quarter, the property "verb" should be "stop"
else if (ordinal >= (numOfEntities - numOfEntities / 4)) {
- assertEquals("stop", String.valueOf(activity.get("verb")));
+ assertEquals("stop", String.valueOf(entity.get("verb")));
}
}
//results should be even ordinals in the first 3 quarters and odd ordinals from the last quarter
@@ -225,17 +225,18 @@ public class AndOrQueryTest extends QueryTestBase {
public void queryWithAndPastLimit() throws IOException {
int numValuesTested = 40;
- generateTestEntities(numValuesTested, "activities");
+ String collectionName = "grapes";
+ generateTestEntities(numValuesTested, collectionName);
//3. Query all entities where "madeup = true"
String errorQuery = "select * where madeup = true";
QueryParameters params = new QueryParameters()
.setQuery(errorQuery)
.setLimit(numValuesTested / 2);//4. Limit the query to half of the number of entities
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//5. Ensure the correct entities are returned
- assertEquals(numValuesTested / 2, activities.getResponse().getEntityCount());
- while (activities.hasNext()) {
- assertTrue(Boolean.parseBoolean(activities.next().get("madeup").toString()));
+ assertEquals(numValuesTested / 2, coll.getResponse().getEntityCount());
+ while (coll.hasNext()) {
+ assertTrue(Boolean.parseBoolean(coll.next().get("madeup").toString()));
}
}
@@ -252,16 +253,17 @@ public class AndOrQueryTest extends QueryTestBase {
public void queryNegated() throws IOException {
int numValuesTested = 20;
- generateTestEntities(numValuesTested, "activities");
+ String collectionName = "huckleberries";
+ generateTestEntities(numValuesTested, collectionName);
//1. Query all entities where "NOT verb = 'go'"
String query = "select * where not verb = 'go'";
//2. Limit the query to half of the number of entities
QueryParameters params = new QueryParameters().setQuery(query).setLimit(numValuesTested / 2);
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. Ensure the returned entities have "verb = 'stop'"
- assertEquals(numValuesTested / 2, activities.getResponse().getEntityCount());
- while (activities.hasNext()) {
- assertEquals("stop", activities.next().get("verb").toString());
+ assertEquals(numValuesTested / 2, coll.getResponse().getEntityCount());
+ while (coll.hasNext()) {
+ assertEquals("stop", coll.next().get("verb").toString());
}
@@ -278,15 +280,16 @@ public class AndOrQueryTest extends QueryTestBase {
public void queryReturnCount() throws Exception {
int numValuesTested = 20;
- generateTestEntities(numValuesTested, "activities");
+ String collectionName = "lemons";
+ generateTestEntities(numValuesTested, collectionName);
//1. Query for a subset of the entities
String inCorrectQuery = "select * where ordinal >= " + (numValuesTested / 2) + " order by ordinal asc";
QueryParameters params = new QueryParameters().setQuery(inCorrectQuery).setLimit(numValuesTested / 2);
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//2. Validate that the correct entities are returned
- assertEquals(numValuesTested / 2, activities.getResponse().getEntityCount());
+ assertEquals(numValuesTested / 2, coll.getResponse().getEntityCount());
- List<Entity> entitiesReturned = activities.getResponse().getEntities();
+ List<Entity> entitiesReturned = coll.getResponse().getEntities();
for (int i = 0; i < numValuesTested / 2; i++) {
assertEquals(numValuesTested / 2 + i, Integer.parseInt(entitiesReturned.get(i).get("ordinal").toString()));
}
@@ -303,7 +306,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryCheckAsc() throws Exception {
int numOfEntities = 20;
- String collectionName = "imagination";
+ String collectionName = "melons";
generateTestEntities(numOfEntities, collectionName);
@@ -332,18 +335,18 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryReturnCheck() throws Exception {
int numOfEntities = 20;
- String collectionName = "imagination";
+ String collectionName = "nectarines";
generateTestEntities(numOfEntities, collectionName);
//2. Issue a query
String inquisitiveQuery = String.format("select * where ordinal >= 0 and ordinal <= %d or WhoHelpedYou = 'Ruff' ORDER BY created", numOfEntities);
QueryParameters params = new QueryParameters().setQuery(inquisitiveQuery);
- Collection activities = this.app().collection(collectionName).get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. validate that a full page of (10) entities is returned
- assertEquals(10, activities.getResponse().getEntityCount());
- List<Entity> entitiesReturned = activities.getResponse().getEntities();
+ assertEquals(10, coll.getResponse().getEntityCount());
+ List<Entity> entitiesReturned = coll.getResponse().getEntities();
for (int i = 0; i < 10; i++) {
assertEquals(i, Integer.parseInt(entitiesReturned.get(i).get("ordinal").toString()));
}
@@ -359,7 +362,7 @@ public class AndOrQueryTest extends QueryTestBase {
@Test
public void queryReturnCheckWithShortHand() throws Exception {
int numOfEntities = 10;
- String collectionName = "imagination";
+ String collectionName = "oranges";
generateTestEntities(numOfEntities, collectionName);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d06dbc95/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
index 0692667..950d59d 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/BadGrammarQueryTest.java
@@ -69,7 +69,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnDoubleQuotes() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "otherthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -95,7 +95,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingQuotes() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "stillotherthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -121,7 +121,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingProperty() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "yetotherthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -147,7 +147,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingPropertyValue() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "thesethings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -173,7 +173,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void exceptionOnMissingOperator() throws IOException {
int numOfEntities = 1;
- String collectionName = "things";
+ String collectionName = "thosethings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -200,7 +200,7 @@ public class BadGrammarQueryTest extends QueryTestBase {
public void limitInQuery() throws IOException {
int numOfEntities =1;
- String collectionName = "things";
+ String collectionName = "whatthings";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d06dbc95/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
index 6591713..e9bb021 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/OrderByTest.java
@@ -41,21 +41,21 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByLongAsc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "beans";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by ordinal asc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
//results should be ordered by ordinal
int index = 0;
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//make sure the correct ordinal properties are returned
- assertEquals(index++, Long.parseLong(activity.get("ordinal").toString()));
+ assertEquals(index++, Long.parseLong(entity.get("ordinal").toString()));
}
}
@@ -68,23 +68,23 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByLongDesc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "chickpeas";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by ordinal desc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
//Since the sort order is descending, start at the last entity we created
int index = numOfEntities - 1;
//results should be sorted by ordinal
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//make sure the correct ordinal properties are returned
//decrement the index to get the next entity in reverse order
- assertEquals(index--, Long.parseLong(activity.get("ordinal").toString()));
+ assertEquals(index--, Long.parseLong(entity.get("ordinal").toString()));
}
}
@@ -96,25 +96,25 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByBooleanAsc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "lentils";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by madeup asc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted false, then true
//The first half of entities returned should have "madeup = false"
//The second half of entities returned should have "madeup = true"
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
if (index++ < numOfEntities / 2) {
- assertEquals("false", activity.get("madeup").toString());
+ assertEquals("false", entity.get("madeup").toString());
} else {
- assertEquals("true", activity.get("madeup").toString());
+ assertEquals("true", entity.get("madeup").toString());
}
}
}
@@ -127,26 +127,26 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByBooleanDesc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "peas";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select * order by madeup desc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted true, then false
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//make sure the booleans are ordered correctly
//The first half of entities returned should have "madeup = true"
//The second half of entities returned should have "madeup = false"
if (index++ < numOfEntities / 2) {
- assertEquals("true", activity.get("madeup").toString());
+ assertEquals("true", entity.get("madeup").toString());
} else {
- assertEquals("false", activity.get("madeup").toString());
+ assertEquals("false", entity.get("madeup").toString());
}
}
}
@@ -159,25 +159,25 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByStringAsc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "carrots";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
//Sort by the "verb" property to test alphabetical sorting of string properties
QueryParameters params = new QueryParameters()
.setQuery("select * order by verb asc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted "go", then "stop"
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//The first half of entities returned should have "verb = 'go'"
//The second half of entities returned should have "verb = 'stop'"
if (index++ < numOfEntities / 2) {
- assertEquals("go", activity.get("verb").toString());
+ assertEquals("go", entity.get("verb").toString());
} else {
- assertEquals("stop", activity.get("verb").toString());
+ assertEquals("stop", entity.get("verb").toString());
}
}
}
@@ -190,7 +190,7 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByStringDesc() throws IOException {
int numOfEntities = 20;
- String collectionName = "activities";
+ String collectionName = "celery";
//create our test entities
generateTestEntities(numOfEntities, collectionName);
@@ -198,18 +198,18 @@ public class OrderByTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery("select * order by verb desc")
.setLimit(numOfEntities);
- Collection activities = this.app().collection("activities").get(params);
- assertEquals(numOfEntities, activities.getResponse().getEntityCount());
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals(numOfEntities, coll.getResponse().getEntityCount());
int index = 0;
//results should be sorted "stop", then "go"
- while (activities.hasNext()) {
- Entity activity = activities.next();
+ while (coll.hasNext()) {
+ Entity entity = coll.next();
//The first half of entities returned should have "verb = 'stop'"
//The second half of entities returned should have "verb = 'go'"
if (index++ < numOfEntities / 2) {
- assertEquals("stop", activity.get("verb").toString());
+ assertEquals("stop", entity.get("verb").toString());
} else {
- assertEquals("go", activity.get("verb").toString());
+ assertEquals("go", entity.get("verb").toString());
}
}
@@ -229,6 +229,7 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByShouldNotAffectResults() throws IOException {
+ String collectionName = "mushrooms";
long created = 0;
Entity actor = new Entity();
actor.put("displayName", "Erin");
@@ -239,10 +240,10 @@ public class OrderByTest extends QueryTestBase {
//1. Insert entities
for (int i = 0; i < 20; i++) {
props.put("ordinal", i);
- Entity activity = this.app().collection("activity").post(props);
- logger.info("Created", activity.get("created").toString());
+ Entity entity = this.app().collection(collectionName).post(props);
+ logger.info("Created", entity.get("created").toString());
if (i == 5) {
- created = Long.parseLong(activity.get("created").toString());
+ created = Long.parseLong(entity.get("created").toString());
}
}
@@ -250,18 +251,18 @@ public class OrderByTest extends QueryTestBase {
//2. Query without 'order by'
String query = "select * where created > " + created;
QueryParameters params = new QueryParameters().setQuery(query);
- Collection activitiesWithoutOrderBy = this.app().collection("activities").get(params);
- assertEquals(10, activitiesWithoutOrderBy.getResponse().getEntityCount());
+ Collection entitiesWithoutOrderBy = this.app().collection(collectionName).get(params);
+ assertEquals(10, entitiesWithoutOrderBy.getResponse().getEntityCount());
//3. Query with 'order by'
query = query + " order by created desc";
params.setQuery(query);
- Collection activitiesWithOrderBy = this.app().collection("activities").get(params);
+ Collection activitiesWithOrderBy = this.app().collection(collectionName).get(params);
assertEquals(10, activitiesWithOrderBy.getResponse().getEntityCount());
//4. Ensure the same entities are returned
- while (activitiesWithoutOrderBy.hasNext() && activitiesWithOrderBy.hasNext()) {
- Entity activityWithoutOrderBy = activitiesWithoutOrderBy.next();
- Entity activityWithOrderBy = activitiesWithOrderBy.next();
- assertEquals(activityWithoutOrderBy.get("uuid").toString(), activityWithOrderBy.get("uuid").toString());
+ while (entitiesWithoutOrderBy.hasNext() && activitiesWithOrderBy.hasNext()) {
+ Entity entityWithoutOrderBy = entitiesWithoutOrderBy.next();
+ Entity entityWithOrderBy = activitiesWithOrderBy.next();
+ assertEquals(entityWithoutOrderBy.get("uuid").toString(), entityWithOrderBy.get("uuid").toString());
}
}
@@ -277,6 +278,7 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByComesBeforeLimitResult() throws IOException {
+ String collectionName = "onions";
Entity actor = new Entity();
actor.put("displayName", "Erin");
Entity props = new Entity();
@@ -286,23 +288,23 @@ public class OrderByTest extends QueryTestBase {
//1. Insert entities
for (int i = 0; i < 20; i++) {
props.put("ordinal", i);
- this.app().collection("activity").post(props);
+ this.app().collection(collectionName).post(props);
}
waitForQueueDrainAndRefreshIndex();
//2. Query a subset of the entities, specifying order and limit
String query = "select * where created > " + 1 + " order by created desc";
QueryParameters params = new QueryParameters().setQuery(query).setLimit(5);
- Collection activities = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. Ensure the correct number of results are returned
- assertEquals(5, activities.getResponse().getEntityCount());
+ assertEquals(5, coll.getResponse().getEntityCount());
//2. Query a subset of the entities, specifying order and limit
query = " where created > " + 1 + " order by created desc";
params = new QueryParameters().setQuery(query).setLimit(5);
- activities = this.app().collection("activities").get(params);
+ coll = this.app().collection(collectionName).get(params);
//3. Ensure the correct number of results are returned
- assertEquals(5, activities.getResponse().getEntityCount());
+ assertEquals(5, coll.getResponse().getEntityCount());
}
/**
@@ -316,8 +318,9 @@ public class OrderByTest extends QueryTestBase {
@Test
public void orderByReturnCorrectResults() throws IOException {
+ String collectionName = "peppers";
int size = 20;
- Entity[] activities = new Entity[size];
+ Entity[] entities = new Entity[size];
Entity actor = new Entity();
actor.put("displayName", "Erin");
@@ -328,34 +331,34 @@ public class OrderByTest extends QueryTestBase {
//1. Insert a number of entities and add them to an array
for (int i = 0; i < size; i++) {
props.put("ordinal", i);
- Entity e = this.app().collection("activity").post(props);
- activities[i] = e;
+ Entity e = this.app().collection(collectionName).post(props);
+ entities[i] = e;
logger.info(String.valueOf(e.get("uuid").toString()));
- logger.info(String.valueOf(Long.parseLong(activities[0].get("created").toString())));
+ logger.info(String.valueOf(Long.parseLong(entities[0].get("created").toString())));
}
waitForQueueDrainAndRefreshIndex(750);
- ArrayUtils.reverse(activities);
- long lastCreated = Long.parseLong(activities[0].get("created").toString());
+ ArrayUtils.reverse(entities);
+ long lastCreated = Long.parseLong(entities[0].get("created").toString());
//2. Query for the entities in descending order
String errorQuery = String.format("select * where created <= %d order by created desc", lastCreated);
int index = 0;
QueryParameters params = new QueryParameters().setQuery(errorQuery);
- Collection activitiesResponse = this.app().collection("activities").get(params);
+ Collection coll = this.app().collection(collectionName).get(params);
//3. Validate that the order is correct
do {
- int returnSize = activitiesResponse.getResponse().getEntityCount();
+ int returnSize = coll.getResponse().getEntityCount();
//loop through the current page of results
for (int i = 0; i < returnSize; i++, index++) {
- assertEquals( ( activities[index] ).get( "uuid" ).toString(),
- activitiesResponse.getResponse().getEntities().get(i).get("uuid").toString());
+ assertEquals( ( entities[index] ).get( "uuid" ).toString(),
+ coll.getResponse().getEntities().get(i).get("uuid").toString());
}
//grab the next page of results
- activitiesResponse = this.app().collection("activities").getNextPage(activitiesResponse, params, true);
+ coll = this.app().collection(collectionName).getNextPage(coll, params, true);
}
- while (activitiesResponse.getCursor() != null);
+ while (coll.getCursor() != null);
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/d06dbc95/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
index acf51c1..cb103cf 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
@@ -41,14 +41,15 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testNestedSelectFieldNames() throws Exception {
- generateTestEntities(20, "things");
+ String collectionName = "basketballs";
+ generateTestEntities(20, collectionName);
QueryParameters params = new QueryParameters()
.setQuery("select actor.displayName,sometestprop where sometestprop = 'testprop'");
- Collection things = this.app().collection("things").get(params);
- assertEquals( 10, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get(params);
+ assertEquals( 10, coll.getNumOfEntities() );
- Iterator<Entity> iter = things.iterator();
+ Iterator<Entity> iter = coll.iterator();
while ( iter.hasNext() ) {
Entity entity = iter.next();
@@ -73,7 +74,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testMixedCaseDupField() throws Exception {
- String collectionName = "things";
+ String collectionName = "baseballs";
String value = RandomStringUtils.randomAlphabetic( 20 );
String otherValue = RandomStringUtils.randomAlphabetic( 20 );
@@ -89,19 +90,19 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + otherValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
params = new QueryParameters()
.setQuery( "select * where TESTPROP='" + otherValue + "'" );
- things = app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
@Test
public void testStringWithSingleQuote() throws Exception {
- String collectionName = "things";
+ String collectionName = "footballs";
String value = "test'value";
String escapedValue = "test\\'value";
@@ -116,14 +117,14 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testprop='" + escapedValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
+ Collection things = this.app().collection(collectionName).get( params );
assertEquals( 1, things.getNumOfEntities() );
}
@Test
public void testStringWithPlus() throws Exception {
- String collectionName = "things";
+ String collectionName = "volleyballs";
String value = "ed+test@usergrid.com";
// create entity with value containing a plus symbol
@@ -135,15 +136,15 @@ public class SelectMappingsQueryTest extends QueryTestBase {
// now query this without encoding the plus symbol
QueryParameters params = new QueryParameters()
.setQuery( "select * where testprop='" + value + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
// again query with the plus symbol url encoded
String escapedValue = "ed%2Btest@usergrid.com";
params = new QueryParameters()
.setQuery( "select * where testprop='" + escapedValue + "'" );
- things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
@@ -154,7 +155,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testFieldOverride1() throws Exception {
- String collectionName = "things";
+ String collectionName = "pickleballs";
// create entity with testProp=value
String value = RandomStringUtils.randomAlphabetic( 20 );
@@ -172,13 +173,13 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + newValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
params = new QueryParameters()
.setQuery( "select * where TESTPROP='" + newValue + "'" );
- things = app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
/**
@@ -187,7 +188,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
@Test
public void testFieldOverride2() throws Exception {
- String collectionName = "things";
+ String collectionName = "tennisballs";
// create entity with TESTPROP=value
String value = RandomStringUtils.randomAlphabetic( 20 );
@@ -205,13 +206,13 @@ public class SelectMappingsQueryTest extends QueryTestBase {
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + newValue + "'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ Collection coll = this.app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
params = new QueryParameters()
.setQuery( "select * where TESTPROP='" + newValue + "'" );
- things = app().collection( "things" ).get( params );
- assertEquals( 1, things.getNumOfEntities() );
+ coll = app().collection(collectionName).get( params );
+ assertEquals( 1, coll.getNumOfEntities() );
}
}
[11/27] usergrid git commit: Basic code improvements 1. Increase some
testing timeouts. 2. improve some logging statements. 3. Fix duplicate
dependency in cored-persistence pom.xml 4. Accept SNS or DISTRIBUTED_SNS as
SNS queue types. 5. Drop missed test
Posted by mr...@apache.org.
Basic code improvements
1. Increase some testing timeouts.
2. improve some logging statements.
3. Fix duplicate dependency in cored-persistence pom.xml
4. Accept SNS or DISTRIBUTED_SNS as SNS queue types.
5. Drop missed test keyspaces for qakka tests.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/400365c3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/400365c3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/400365c3
Branch: refs/heads/expose-reindex
Commit: 400365c36eb1a46ba76ca3b2cf8f2db6670227b1
Parents: 25f1b0e
Author: Mike Dunker <md...@google.com>
Authored: Thu Aug 17 08:59:18 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 13:37:06 2017 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
.../actorsystem/ActorSystemManagerImpl.java | 3 ++-
.../persistence/index/impl/EsEntityIndexImpl.java | 2 +-
stack/corepersistence/queue/pom.xml | 18 ++++++------------
.../persistence/queue/guice/QueueModule.java | 2 +-
.../persistence/qakka/KeyspaceDropper.java | 6 +++++-
.../qakka/core/QueueMessageManagerTest.java | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
9 files changed, 19 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 77b047e..bf1f5e7 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -112,7 +112,7 @@ cassandra.embedded=false
# Read timeout for an individual request (in millseconds)
#
-#cassandra.timeout=5000
+#cassandra.timeout=20000
# Set the credentials used for Cassandra, if any.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index e32b146..df253f0 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -16,7 +16,7 @@
# with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml)
#cassandra.connections=30
cassandra.timeout.pool=20000
-cassandra.timeout=20000
+cassandra.timeout=25000
#Not a good number for real systems. Write shards should be 2x cluster size from our tests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
index bfa3abe..ea9ada8 100644
--- a/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
+++ b/stack/corepersistence/actorsystem/src/main/java/org/apache/usergrid/persistence/actorsystem/ActorSystemManagerImpl.java
@@ -377,6 +377,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
Iterator<Class> messageTypes = routerProducer.getMessageTypes().iterator();
while ( messageTypes.hasNext() ) {
Class messageType = messageTypes.next();
+ logger.info("createClusterSystem: routerProducer {}: message type={}", routerProducer.getRouterPath(), messageType.getName());
routersByMessageType.put( messageType, routerProducer.getRouterPath() );
}
}
@@ -467,7 +468,7 @@ public class ActorSystemManagerImpl implements ActorSystemManager {
if (started) {
logger.info( "ClientActor [{}] has started", ra.path() );
} else {
- throw new RuntimeException( "ClientActor ["+ra.path()+"] did not start in time" );
+ throw new RuntimeException( "ClientActor ["+ra.path()+"] did not start in time, validate that akka seeds are configured properly" );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index a35921c..cb695d5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -502,7 +502,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
searchResponse = srb.execute().actionGet();
}
catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t.getMessage() );
+ logger.error( "Unable to communicate with Elasticsearch: {}", t.getMessage() );
failureMonitor.fail( "Unable to execute batch", t );
throw t;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queue/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/pom.xml b/stack/corepersistence/queue/pom.xml
index 153ed4b..005ce0c 100644
--- a/stack/corepersistence/queue/pom.xml
+++ b/stack/corepersistence/queue/pom.xml
@@ -93,6 +93,12 @@
<groupId>org.apache.usergrid</groupId>
<artifactId>common</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>cassandra-all</artifactId>
+ <groupId>org.apache.cassandra</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
@@ -152,18 +158,6 @@
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.usergrid</groupId>
- <artifactId>common</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <artifactId>cassandra-all</artifactId>
- <groupId>org.apache.cassandra</groupId>
- </exclusion>
- </exclusions>
- </dependency>
-
<!--
<dependency>
<groupId>com.datastax.cassandra</groupId>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
index a485f55..a6e2451 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/guice/QueueModule.java
@@ -43,7 +43,7 @@ public class QueueModule extends AbstractModule {
public QueueModule( String queueManagerType ) {
logger.info("QueueManagerType={}", queueManagerType);
- if ( "DISTRIBUTED_SNS".equals( queueManagerType ) ) {
+ if ( "DISTRIBUTED_SNS".equals( queueManagerType ) || "SNS".equals(queueManagerType)) {
this.implementation = LegacyQueueManager.Implementation.DISTRIBUTED_SNS;
}
else if ( "DISTRIBUTED".equals( queueManagerType ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java
index e220650..b2cebaa 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/KeyspaceDropper.java
@@ -34,7 +34,7 @@ import java.util.Properties;
*/
public class KeyspaceDropper {
- private static final Logger logger = LoggerFactory.getLogger( AbstractTest.class );
+ private static final Logger logger = LoggerFactory.getLogger( KeyspaceDropper.class );
static { dropTestKeyspaces(); }
@@ -57,6 +57,10 @@ public class KeyspaceDropper {
dropTestKeyspace( keyspaceApp, hosts, port );
dropTestKeyspace( keyspaceQueue, hosts, port );
+
+ // drop local test keyspaces
+ dropTestKeyspace(keyspaceApp + "_", hosts, port);
+ dropTestKeyspace(keyspaceQueue + "_", hosts, port);
}
public static void dropTestKeyspace( String keyspace, String[] hosts, int port ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java
index d4ed7ef..8da2180 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/qakka/core/QueueMessageManagerTest.java
@@ -340,7 +340,7 @@ public class QueueMessageManagerTest extends AbstractAkkaTest {
if (qmm.getQueueDepth( queueName, available ) == numMessages) {
break;
}
- Thread.sleep( 500 );
+ Thread.sleep( 1000 );
}
Assert.assertEquals( numMessages, qmm.getQueueDepth( queueName, available ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/400365c3/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index cbec81f..4b76ca4 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -18,7 +18,7 @@
cassandra.startup=external
cassandra.connections=30
cassandra.timeout.pool=20000
-cassandra.timeout=20000
+cassandra.timeout=25000
hystrix.threadpool.graph_user.coreSize=1200
[10/27] usergrid git commit: add a little more wait time
Posted by mr...@apache.org.
add a little more wait time
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/25f1b0e5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/25f1b0e5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/25f1b0e5
Branch: refs/heads/expose-reindex
Commit: 25f1b0e5a86f51d0da465d78cb0c5fbd1a006a7d
Parents: d06dbc9
Author: Mike Dunker <md...@google.com>
Authored: Wed Aug 16 22:30:34 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 11:59:53 2017 -0700
----------------------------------------------------------------------
.../apache/usergrid/rest/applications/queries/QueryTestBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/25f1b0e5/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
index 0adafef..1eec9ae 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
@@ -65,7 +65,7 @@ public class QueryTestBase extends AbstractRestIT {
logger.info(entities[i].entrySet().toString());
}
//refresh the index so that they are immediately searchable
- this.waitForQueueDrainAndRefreshIndex();
+ this.waitForQueueDrainAndRefreshIndex(1000);
return entities;
}
[08/27] usergrid git commit: more cassandra.timeout increases for
tests
Posted by mr...@apache.org.
more cassandra.timeout increases for tests
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4b916c4d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4b916c4d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4b916c4d
Branch: refs/heads/expose-reindex
Commit: 4b916c4d39fa0018fbe01f8e0489b0f5d083e41e
Parents: 70de6fd
Author: Mike Dunker <md...@google.com>
Authored: Thu Aug 17 11:57:53 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 11:57:53 2017 -0700
----------------------------------------------------------------------
stack/core/src/test/resources/usergrid-custom-test.properties | 1 +
.../collection/src/test/resources/dynamic-test.properties | 1 +
.../collection/src/test/resources/usergrid-CHOP.properties | 1 +
.../collection/src/test/resources/usergrid-UNIT.properties | 1 +
.../collection/src/test/resources/usergrid.properties | 1 +
.../common/src/test/resources/usergrid-UNIT.properties | 2 +-
.../graph/src/test/resources/usergrid-AWS.properties | 1 +
.../graph/src/test/resources/usergrid-CHOP.properties | 1 +
.../graph/src/test/resources/usergrid-UNIT.properties | 1 +
stack/corepersistence/graph/src/test/resources/usergrid.properties | 1 +
.../queryindex/src/test/resources/dynamic-test.properties | 1 +
.../queryindex/src/test/resources/usergrid-CHOP.properties | 1 +
.../queryindex/src/test/resources/usergrid-UNIT.properties | 1 +
stack/corepersistence/queue/src/test/resources/qakka.properties | 1 +
stack/rest/src/test/resources/usergrid-custom-test.properties | 1 +
15 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index 8f9058d..e32b146 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -16,6 +16,7 @@
# with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml)
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
#Not a good number for real systems. Write shards should be 2x cluster size from our tests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
index ca1a51c..59b8d31 100644
--- a/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
+++ b/stack/corepersistence/collection/src/test/resources/dynamic-test.properties
@@ -2,6 +2,7 @@
# safe dynamic property defaults for our testing via IDE or Maven
cassandra.connections=30
cassandra.timeout.pool=30000
+cassandra.timeout=30000
cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
index c82bf83..9e95663 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-CHOP.properties
@@ -20,6 +20,7 @@
cassandra.connections=30
cassandra.timeout.pool=30000
+cassandra.timeout=30000
cassandra.port=9160
# a comma delimited private IP address list to your chop cassandra cluster
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
index 82bea14..0253e6e 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
@@ -21,6 +21,7 @@
# purposely setting connections lower to slow down the activity on the cassandra server locally
cassandra.connections=5
cassandra.timeout.pool=60000
+cassandra.timeout=60000
cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/collection/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid.properties b/stack/corepersistence/collection/src/test/resources/usergrid.properties
index e85b1ac..b7a6c11 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid.properties
@@ -33,6 +33,7 @@ collection.uniquevalues.authoritative.region=us-east
cassandra.connections=15
cassandra.timeout.pool=240000
+cassandra.timeout=240000
queue.num.actors=5
queue.sender.num.actors=5
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
index 0a43209..13b2937 100644
--- a/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/common/src/test/resources/usergrid-UNIT.properties
@@ -24,7 +24,7 @@ cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
collections.keyspace=Usergrid_Collections
-cassandra.timeout=10000
+cassandra.timeout=20000
cassandra.embedded=true
collections.keyspace.strategy.options=replication_factor:1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
index 4dd17e2..63da408 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid-AWS.properties
@@ -1,6 +1,7 @@
# Keep nothing but overriding test defaults in here
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
cassandra.hosts=
#cassandra.hosts=localhost
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties
index ea4c3c7..9680d88 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid-CHOP.properties
@@ -2,6 +2,7 @@
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
# a comma delimited private IP address list to your chop cassandra cluster
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
index 96834e6..3cb5608 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
@@ -1,6 +1,7 @@
# Keep nothing but overriding test defaults in here
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/graph/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid.properties b/stack/corepersistence/graph/src/test/resources/usergrid.properties
index d744efe..f8b7a8d 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid.properties
@@ -3,3 +3,4 @@ usergrid.cluster_name=usergrid
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
index 2589a7f..aaa4653 100644
--- a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
+++ b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
@@ -2,6 +2,7 @@
# safe dynamic property defaults for our testing via IDE or Maven
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties
index 677df96..855a8b5 100644
--- a/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties
+++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-CHOP.properties
@@ -1,6 +1,7 @@
# These are for CHOP environment settings
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
cassandra.port=9160
# a comma delimited private IP address list to your chop cassandra cluster
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
index 67a9ab1..c0166e7 100644
--- a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
@@ -6,6 +6,7 @@ cassandra.port=9160
cassandra.cluster_name=Usergrid
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
collections.keyspace=Usergrid_Collections
collections.keyspace.strategy.options=replication_factor:1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/corepersistence/queue/src/test/resources/qakka.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/resources/qakka.properties b/stack/corepersistence/queue/src/test/resources/qakka.properties
index d77e7e8..2a1a83e 100644
--- a/stack/corepersistence/queue/src/test/resources/qakka.properties
+++ b/stack/corepersistence/queue/src/test/resources/qakka.properties
@@ -67,3 +67,4 @@ cassandra.keyspace-drop-and-create=true
#cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b916c4d/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 615bedd..cbec81f 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -18,6 +18,7 @@
cassandra.startup=external
cassandra.connections=30
cassandra.timeout.pool=20000
+cassandra.timeout=20000
hystrix.threadpool.graph_user.coreSize=1200
[22/27] usergrid git commit: Bypass check of app APM setting if APM
not enabled
Posted by mr...@apache.org.
Bypass check of app APM setting if APM not enabled
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9e16e2f0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9e16e2f0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9e16e2f0
Branch: refs/heads/expose-reindex
Commit: 9e16e2f006676d9e16997d378224e06a8683c744
Parents: 4170295
Author: Mike Dunker <md...@google.com>
Authored: Thu Sep 14 15:07:30 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Sep 14 15:07:30 2017 -0700
----------------------------------------------------------------------
.../apache/usergrid/rest/applications/ApplicationResource.java | 4 ++++
.../java/org/apache/usergrid/management/ManagementService.java | 2 ++
.../usergrid/management/cassandra/ManagementServiceImpl.java | 5 +++++
3 files changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e16e2f0/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
index 7479a90..a6eefb1 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/ApplicationResource.java
@@ -621,6 +621,10 @@ public class ApplicationResource extends CollectionResource {
public Object getAPMConfig( @Context UriInfo ui,
@QueryParam("callback") @DefaultValue("callback") String callback )
throws Exception {
+ if (!management.isAPMEnabled()) {
+ // no need to check app for setting
+ throw new EntityNotFoundException("apigeeMobileConfig not enabled.");
+ }
EntityManager em = emf.getEntityManager( applicationId );
Object value = em.getProperty( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ),
APIGEE_MOBILE_APM_CONFIG_JSON_KEY );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e16e2f0/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 8b840d6..3d0c9fb 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -343,6 +343,8 @@ public interface ManagementService {
Object registerAppWithAPM( OrganizationInfo orgInfo, ApplicationInfo appInfo ) throws Exception;
+ boolean isAPMEnabled() throws Exception;
+
/** For testing purposes only */
Properties getProperties();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9e16e2f0/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 89375fd..2302d8d 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -3404,6 +3404,11 @@ public class ManagementServiceImpl implements ManagementService {
}
@Override
+ public boolean isAPMEnabled() throws Exception {
+ return false;
+ }
+
+ @Override
public Observable<Id> deleteAllEntities(final UUID applicationId,final int limit){
if(applicationId.equals(CpNamingUtils.MANAGEMENT_APPLICATION_ID)){
throw new IllegalArgumentException("Can't delete from management app");
[06/27] usergrid git commit: fix broken recaptcha
Posted by mr...@apache.org.
fix broken recaptcha
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/459163b8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/459163b8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/459163b8
Branch: refs/heads/expose-reindex
Commit: 459163b8ddb4b80ebb43273a8646ee9c8a6fe2bd
Parents: be2f69c
Author: Mike Dunker <md...@google.com>
Authored: Wed Aug 9 18:39:23 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 09:47:40 2017 -0700
----------------------------------------------------------------------
.../rest/applications/ApplicationResource/authorize_form.jsp | 2 +-
.../rest/applications/users/UserResource/resetpw_email_form.jsp | 4 ++--
.../rest/applications/users/UserResource/resetpw_set_form.jsp | 2 +-
.../rest/applications/users/UsersResource/resetpw_email_form.jsp | 4 ++--
.../rest/management/ManagementResource/authorize_form.jsp | 2 +-
.../rest/management/users/UserResource/resetpw_email_form.jsp | 4 ++--
.../rest/management/users/UserResource/resetpw_set_form.jsp | 2 +-
.../rest/management/users/UsersResource/resetpw_email_form.jsp | 4 ++--
8 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
index 0079bcf..ed934a7 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
@@ -29,7 +29,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<input type="hidden" name="response_type" value="${fn:escapeXml(it.responseType)}">
<input type="hidden" name="client_id" value="${fn:escapeXml(it.clientId)}">
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
index 59026bf..5230ea7 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <c:if test="${!empty it.errorMsg}">
<div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
@@ -37,7 +37,7 @@ limitations under the License.
<c:out value="${it.user.email}" escapeXml="true" />
</p>
<p id="human-proof"></p>
- ${fn:escapeXml(it.reCaptchaHtml)}
+ ${it.reCaptchaHtml}
<p class="buttons">
<input type="submit" value="submit" />
</p>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
index 19b6528..c0203ce 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
@@ -31,7 +31,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<input type="hidden" name="token" value="${fn:escapeXml(it.token)}">
<fieldset>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
index f86240f..01dfc57 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
@@ -28,7 +28,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <c:if test="${!empty it.errorMsg}">
<div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
@@ -42,7 +42,7 @@ limitations under the License.
<input class="text_field" id="email" name="email" type="text" />
</p>
<p id="human-proof"></p>
- ${fn:escapeXml(it.reCaptchaHtml)}
+ ${it.reCaptchaHtml}
<p class="buttons">
<button type="submit">Submit</button>
</p>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
index 0079bcf..ed934a7 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
@@ -29,7 +29,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<input type="hidden" name="response_type" value="${fn:escapeXml(it.responseType)}">
<input type="hidden" name="client_id" value="${fn:escapeXml(it.clientId)}">
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
index c9f8309..6341d60 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
<div class="dialog-area password-reset-form">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <c:if test="${!empty it.errorMsg}">
<div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
@@ -37,7 +37,7 @@ limitations under the License.
<c:out value="${it.user.email}" escapeXml="true" />
</p>
<p id="human-proof"></p>
- ${fn:escapeXml(it.reCaptchaHtml)}
+ ${it.reCaptchaHtml}
<p class="buttons">
<input type="submit" value="submit" />
</p>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
index 60384c4..6334466 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
@@ -30,7 +30,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<input type="hidden" name="token" value="${fn:escapeXml(it.token)}">
<fieldset>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/459163b8/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
index 8b15cd4..f8cf496 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
@@ -27,7 +27,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty fn:escnapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
+ <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
<p>
@@ -38,7 +38,7 @@ limitations under the License.
<input class="text_field" id="email" name="email" type="text" />
</p>
<p id="human-proof"></p>
- ${fn:escapeXml(it.reCaptchaHtml)}
+ ${it.reCaptchaHtml}
<p class="buttons">
<button type="submit">Submit</button>
</p>
[15/27] usergrid git commit: Converts de-indexing of edges and
entities to identify documents in the index to delete based on data in
Cassandra vs. resource intensive queries to the index. Fixes issue where
nodes were not actually getting deleted from gr
Posted by mr...@apache.org.
Converts de-indexing of edges and entities to identify documents in the index to delete based on data in Cassandra vs. resource intensive queries to the index. Fixes issue where nodes were not actually getting deleted from graph as marked edges were being filtered out during the delete process itself. Update to a newer vs. of jamm (used for jvm memory management in the test framework).
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c201f1f6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c201f1f6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c201f1f6
Branch: refs/heads/expose-reindex
Commit: c201f1f67aa0f373958c3886c893979215cba88f
Parents: 570e1ab
Author: Michael Russo <ru...@google.com>
Authored: Sat Jul 8 17:10:25 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Fri Aug 18 17:00:43 2017 -0700
----------------------------------------------------------------------
.../corepersistence/EntityManagerFig.java | 2 +-
.../asyncevents/AsyncEventServiceImpl.java | 23 +--
.../asyncevents/EventBuilder.java | 15 +-
.../asyncevents/EventBuilderImpl.java | 172 ++++++++++++++-----
.../corepersistence/index/IndexService.java | 16 +-
.../corepersistence/index/IndexServiceImpl.java | 56 ++----
.../read/traverse/AbstractReadGraphFilter.java | 37 +---
.../AbstractReadReverseGraphFilter.java | 36 +---
.../org/apache/usergrid/persistence/Query.java | 17 ++
.../rx/EdgesFromSourceObservableIT.java | 2 +-
.../rx/EdgesToTargetObservableIT.java | 4 +-
.../core/src/test/resources/project.properties | 2 +-
.../persistence/graph/GraphManager.java | 2 +-
.../graph/impl/GraphManagerImpl.java | 21 +--
.../graph/impl/SimpleSearchByEdgeType.java | 9 +
.../graph/impl/stage/NodeDeleteListener.java | 6 +-
.../impl/stage/NodeDeleteListenerImpl.java | 64 +++----
.../graph/serialization/EdgesObservable.java | 6 +-
.../impl/EdgeMetadataSerializationV2Impl.java | 1 -
.../serialization/impl/EdgesObservableImpl.java | 22 +--
.../impl/TargetIdObservableImpl.java | 4 +-
.../impl/migration/EdgeDataMigrationImpl.java | 2 +-
.../graph/impl/NodeDeleteListenerTest.java | 8 +-
.../usergrid/persistence/index/EntityIndex.java | 12 +-
.../index/impl/EsEntityIndexImpl.java | 63 -------
.../persistence/index/query/Identifier.java | 3 +-
stack/pom.xml | 8 +-
stack/rest/pom.xml | 4 +-
.../rest/src/test/resources/project.properties | 2 +-
stack/services/pom.xml | 4 +-
.../src/test/resources/project.properties | 2 +-
stack/test-utils/pom.xml | 4 +-
.../src/test/resources/project.properties | 2 +-
stack/tools/pom.xml | 2 +-
stack/websocket/pom.xml | 2 +-
35 files changed, 286 insertions(+), 349 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
index 872ffbb..46c7a1d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/EntityManagerFig.java
@@ -39,7 +39,7 @@ public interface EntityManagerFig extends GuicyFig {
int sleep();
@Key( "usergrid.entityManager.enable_deindex_on_update" )
- @Default( "true" )
+ @Default( "false" )
boolean getDeindexOnUpdate();
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 0e55e9b..79a80c0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -640,8 +640,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
// default this observable's return to empty index operation message if nothing is emitted
- return eventBuilder.buildDeleteEdge(applicationScope, edge)
- .toBlocking().lastOrDefault(new IndexOperationMessage());
+ return eventBuilder.buildDeleteEdge(applicationScope, edge);
}
@@ -832,20 +831,11 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final ApplicationScope applicationScope = entityDeleteEvent.getEntityIdScope().getApplicationScope();
final Id entityId = entityDeleteEvent.getEntityIdScope().getId();
- if (logger.isDebugEnabled())
+ if (logger.isDebugEnabled()) {
logger.debug("Deleting entity id from index in app scope {} with entityId {}", applicationScope, entityId);
+ }
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete( applicationScope, entityId );
-
-
- // Delete the entities and remove from graph separately
- entityDeleteResults.getEntitiesDeleted().toBlocking().lastOrDefault(null);
-
- entityDeleteResults.getCompactedNode().toBlocking().lastOrDefault(null);
-
- // default this observable's return to empty index operation message if nothing is emitted
- return entityDeleteResults.getIndexObservable().toBlocking().lastOrDefault(new IndexOperationMessage());
+ return eventBuilder.buildEntityDelete( applicationScope, entityId );
}
@@ -885,9 +875,6 @@ public class AsyncEventServiceImpl implements AsyncEventService {
startWorker(AsyncEventQueueType.UTILITY);
}
- for (int i = 0; i < deleteCount; i++) {
- startWorker(AsyncEventQueueType.DELETE);
- }
if( indexQueue instanceof SNSQueueManagerImpl) {
logger.info("Queue manager implementation supports dead letters, start dead letter queue workers.");
for (int i = 0; i < indexDeadCount; i++) {
@@ -902,7 +889,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
startDeadQueueWorker(AsyncEventQueueType.DELETE);
}
}else{
- logger.info("Queue manager implementation does NOT support dead letters, NOT starting dead letter queue worker.");
+ logger.info("Queue manager implementation does NOT support dead letters, NOT starting dead letter queue workers.");
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
index 4db9f4b..ebb9190 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
@@ -27,6 +27,7 @@ import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.index.impl.IndexOperationMessage;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -54,7 +55,7 @@ public interface EventBuilder {
* @param edge
* @return
*/
- Observable<IndexOperationMessage> buildDeleteEdge( ApplicationScope applicationScope, Edge edge );
+ IndexOperationMessage buildDeleteEdge( ApplicationScope applicationScope, Edge edge );
/**
* Return a bin with 2 observable streams for entity delete.
@@ -62,7 +63,7 @@ public interface EventBuilder {
* @param entityId
* @return
*/
- EntityDeleteResults buildEntityDelete(ApplicationScope applicationScope, Id entityId );
+ IndexOperationMessage buildEntityDelete(ApplicationScope applicationScope, Id entityId );
@@ -94,17 +95,17 @@ public interface EventBuilder {
- private final Observable<Id> compactedNode;
+ private final Observable<MarkedEdge> deletedEdges;
public EntityDeleteResults( final Observable<IndexOperationMessage> indexOperationMessageObservable,
final Observable<List<MvccLogEntry>> entitiesDeleted,
- final Observable<Id> compactedNode) {
+ final Observable<MarkedEdge> deletedEdges) {
this.indexOperationMessageObservable = indexOperationMessageObservable;
this.entitiesDeleted = entitiesDeleted;
- this.compactedNode = compactedNode;
+ this.deletedEdges = deletedEdges;
}
@@ -116,8 +117,8 @@ public interface EventBuilder {
return entitiesDeleted;
}
- public Observable<Id> getCompactedNode() {
- return compactedNode;
+ public Observable<MarkedEdge> getEdgesDeleted() {
+ return deletedEdges;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index bbdce5a..5051598 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -24,12 +24,11 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import org.apache.usergrid.corepersistence.index.*;
import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.corepersistence.index.EntityIndexOperation;
-import org.apache.usergrid.corepersistence.index.IndexService;
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
@@ -82,7 +81,7 @@ public class EventBuilderImpl implements EventBuilder {
if (logger.isDebugEnabled()) {
logger.debug("Indexing in app scope {} with entity {} and new edge {}",
- applicationScope, entity, newEdge);
+ applicationScope, entity, newEdge);
}
return indexService.indexEdge( applicationScope, entity, newEdge );
@@ -90,15 +89,38 @@ public class EventBuilderImpl implements EventBuilder {
@Override
- public Observable<IndexOperationMessage> buildDeleteEdge( final ApplicationScope applicationScope, final Edge
+ public IndexOperationMessage buildDeleteEdge( final ApplicationScope applicationScope, final Edge
edge ) {
if (logger.isDebugEnabled()) {
logger.debug("Deleting in app scope {} with edge {}", applicationScope, edge);
}
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- return gm.deleteEdge( edge )
- .flatMap( deletedEdge -> indexService.deleteIndexEdge( applicationScope, deletedEdge ));
+ final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+ final IndexOperationMessage combined = new IndexOperationMessage();
+
+ gm.deleteEdge( edge )
+ .doOnNext( deletedEdge -> {
+
+ logger.debug("Processing deleted edge for de-indexing {}", deletedEdge);
+
+ // get ALL versions of the target node as any connection from this source node needs to be removed
+ ecm.getVersionsFromMaxToMin(deletedEdge.getTargetNode(), UUIDUtils.newTimeUUID())
+ .doOnNext(mvccLogEntry -> {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Adding edge {} mvccLogEntry {} to de-index batch", deletedEdge.getTargetNode(), mvccLogEntry);
+ }
+ combined.ingest(
+ indexService
+ .deIndexEdge(applicationScope, deletedEdge, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion())
+ .toBlocking().lastOrDefault(new IndexOperationMessage()));
+
+ }).toBlocking().lastOrDefault(null);
+
+ }).toBlocking().lastOrDefault(null);
+
+ return combined;
}
@@ -106,43 +128,102 @@ public class EventBuilderImpl implements EventBuilder {
//it'll need to be pushed up higher so we can do the marking that isn't async or does it not matter?
@Override
- public EntityDeleteResults buildEntityDelete(final ApplicationScope applicationScope, final Id entityId ) {
+ public IndexOperationMessage buildEntityDelete(final ApplicationScope applicationScope, final Id entityId ) {
+
if (logger.isDebugEnabled()) {
- logger.debug("Deleting entity id from index in app scope {} with entityId {}", applicationScope, entityId);
+ logger.debug("Deleting entity id (marked versions) from index in app scope {} with entityId {}",
+ applicationScope, entityId);
}
final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
- //TODO USERGRID-1123: Implement so we don't iterate logs twice (latest DELETED version, then to get all DELETED)
-
- MvccLogEntry mostRecentlyMarked = ecm.getVersionsFromMaxToMin( entityId, UUIDUtils.newTimeUUID() ).toBlocking()
- .firstOrDefault( null, mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
-
- // De-indexing and entity deletes don't check log entries. We must do that first. If no DELETED logs, then
- // return an empty observable as our no-op.
- Observable<IndexOperationMessage> deIndexObservable = Observable.empty();
- Observable<List<MvccLogEntry>> ecmDeleteObservable = Observable.empty();
-
- if(mostRecentlyMarked != null){
+ MvccLogEntry mostRecentToDelete =
+ ecm.getVersionsFromMaxToMin( entityId, UUIDUtils.newTimeUUID() )
+ .toBlocking()
+ .firstOrDefault( null, mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
- // fetch entity versions to be de-index by looking in cassandra
- deIndexObservable =
- indexService.deIndexEntity(applicationScope, entityId, mostRecentlyMarked.getVersion(),
- getVersionsOlderThanMarked(ecm, entityId, mostRecentlyMarked.getVersion()));
+// logger.info("mostRecent stage={} entityId={} version={} state={}",
+// mostRecentToDelete.getStage().name(), mostRecentToDelete.getEntityId(),
+// mostRecentToDelete.getVersion().toString(), mostRecentToDelete.getState().name());
- ecmDeleteObservable =
- ecm.getVersionsFromMaxToMin( entityId, mostRecentlyMarked.getVersion() )
- .filter( mvccLogEntry->
- mvccLogEntry.getVersion().timestamp() <= mostRecentlyMarked.getVersion().timestamp() )
- .buffer( serializationFig.getBufferSize() )
- .doOnNext( buffer -> ecm.delete( buffer ) );
+ if (mostRecentToDelete == null) {
+ logger.info("No entity versions to delete for id {}", entityId.toString());
+ }
+ // if nothing is marked, then abort
+ if(mostRecentToDelete == null){
+ return new IndexOperationMessage();
}
- // Graph compaction checks the versions inside compactNode, just build this up for the caller to subscribe to
- final Observable<Id> graphCompactObservable = gm.compactNode(entityId);
-
- return new EntityDeleteResults( deIndexObservable, ecmDeleteObservable, graphCompactObservable );
+ final List<MvccLogEntry> logEntries = new ArrayList<>();
+ Observable<MvccLogEntry> mvccLogEntryListObservable =
+ ecm.getVersionsFromMaxToMin( entityId, UUIDUtils.newTimeUUID() );
+ mvccLogEntryListObservable
+ .filter( mvccLogEntry-> mvccLogEntry.getVersion().timestamp() <= mostRecentToDelete.getVersion().timestamp() )
+ .buffer( serializationFig.getBufferSize() )
+ .doOnNext( buffer -> ecm.delete( buffer ) )
+ .doOnNext(mvccLogEntries -> {
+ logEntries.addAll(mvccLogEntries);
+ }).toBlocking().lastOrDefault(null);
+
+ //logger.info("logEntries size={}", logEntries.size());
+
+ IndexOperationMessage combined = new IndexOperationMessage();
+
+ // do the edge deletes and build up de-index messages for each edge deleted
+ // assume we have "server1" and "region1" nodes in the graph with the following relationships (edges/connections):
+ //
+ // region1 -- zzzconnzzz|has --> server1
+ // server1 -- zzzconnzzz|in --> region1
+ //
+ // there will always be a relationship from the appId to each entity based on the entity type (collection):
+ //
+ // application -- zzzcollzzz|servers --> server1
+ // application -- zzzcollzzz|regions --> region1
+ //
+ // When deleting either "server1" or "region1" entity, the connections should get deleted and de-indexed along
+ // with the entry for the entity itself in the collection. The above example should have at minimum 3 things to
+ // be de-indexed. There may be more as either "server1" or "region1" could have multiple versions.
+ //
+ // Further comments using the example of deleting "server1" from the above example.
+ gm.compactNode(entityId).doOnNext(markedEdge -> {
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Processing deleted edge for de-indexing {}", markedEdge);
+ }
+
+ // if the edge was for a connection where the entity to be deleted is the source node, we need to load
+ // the target node's versions so that all versions of connections to that entity can be de-indexed
+ // server1 -- zzzconnzzz|in --> region1
+ if(!markedEdge.getTargetNode().getType().equals(entityId.getType())){
+
+ // get ALL versions of the target node as any connection from this source node needs to be removed
+ ecm.getVersionsFromMaxToMin( markedEdge.getTargetNode(), UUIDUtils.newTimeUUID() )
+ .doOnNext(mvccLogEntry -> {
+ logger.debug("Adding edge {} mvccLogEntry {} to de-index batch", markedEdge, mvccLogEntry);
+ combined.ingest(
+ indexService
+ .deIndexEdge(applicationScope, markedEdge, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion())
+ .toBlocking().lastOrDefault(new IndexOperationMessage()));
+
+ }).toBlocking().lastOrDefault(null);
+
+ }else {
+
+ // for each version of the entity being deleted, de-index the connections where the entity is the target
+ // node ( application -- zzzcollzzz|servers --> server1 ) or (region1 -- zzzconnzzz|has --> server1)
+ logEntries.forEach(logEntry -> {
+ logger.debug("Adding edge {} mvccLogEntry {} to de-index batch", markedEdge, logEntry);
+ combined.ingest(
+ indexService
+ .deIndexEdge(applicationScope, markedEdge, logEntry.getEntityId(), logEntry.getVersion())
+ .toBlocking().lastOrDefault(new IndexOperationMessage()));
+ });
+ }
+
+ }).toBlocking().lastOrDefault(null);
+
+ return combined;
}
@Override
@@ -185,22 +266,22 @@ public class EventBuilderImpl implements EventBuilder {
return indexService.deIndexOldVersions( applicationScope, entityId,
- getVersionsOlderThanMarked(ecm, entityId, markedVersion), markedVersion);
+ getVersionsOlderThanOrEqualToMarked(ecm, entityId, markedVersion));
}
- private List<UUID> getVersionsOlderThanMarked( final EntityCollectionManager ecm,
- final Id entityId, final UUID markedVersion ){
+ private List<UUID> getVersionsOlderThanOrEqualToMarked(final EntityCollectionManager ecm,
+ final Id entityId, final UUID markedVersion ){
final List<UUID> versions = new ArrayList<>();
- // only take last 5 versions to avoid eating memory. a tool can be built for massive cleanups for old usergrid
+ // only take last 100 versions to avoid eating memory. a tool can be built for massive cleanups for old usergrid
// clusters that do not have this in-line cleanup
ecm.getVersionsFromMaxToMin( entityId, markedVersion)
- .take(5)
+ .take(100)
.forEach( mvccLogEntry -> {
- if ( mvccLogEntry.getVersion().timestamp() < markedVersion.timestamp() ) {
+ if ( mvccLogEntry.getVersion().timestamp() <= markedVersion.timestamp() ) {
versions.add(mvccLogEntry.getVersion());
}
@@ -210,4 +291,17 @@ public class EventBuilderImpl implements EventBuilder {
return versions;
}
+ private List<UUID> getAllVersions( final EntityCollectionManager ecm,
+ final Id entityId ) {
+
+ final List<UUID> versions = new ArrayList<>();
+
+ ecm.getVersionsFromMaxToMin(entityId, UUIDUtils.newTimeUUID())
+ .forEach( mvccLogEntry -> {
+ versions.add(mvccLogEntry.getVersion());
+ });
+
+ return versions;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
index b989a9c..58d470a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexService.java
@@ -69,18 +69,17 @@ public interface IndexService {
*/
Observable<IndexOperationMessage> deleteIndexEdge(final ApplicationScope applicationScope, final Edge edge);
-
/**
- * De-index all documents with the specified entityId and versions provided. This will also remove any documents
- * where the entity is a source/target node ( index docs where this entityId is a part of connections).
- *
+ * Delete an index edge from the specified scope for a specific entity version
* @param applicationScope
+ * @param edge
* @param entityId
- * @param markedVersion
+ * @param entityVersion
* @return
*/
- Observable<IndexOperationMessage> deIndexEntity(final ApplicationScope applicationScope, final Id entityId,
- final UUID markedVersion, final List<UUID> allVersionsBeforeMarked);
+ Observable<IndexOperationMessage> deIndexEdge(final ApplicationScope applicationScope, final Edge edge,
+ final Id entityId, final UUID entityVersion);
+
/**
@@ -88,10 +87,9 @@ public interface IndexService {
*
* @param applicationScope
* @param entityId
- * @param markedVersion
* @return
*/
Observable<IndexOperationMessage> deIndexOldVersions(final ApplicationScope applicationScope, final Id entityId,
- final List<UUID> versions, UUID markedVersion);
+ final List<UUID> versions);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 1b8614f..32470f6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -29,9 +29,7 @@ import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.*;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.index.*;
@@ -103,7 +101,7 @@ public class IndexServiceImpl implements IndexService {
//we always index in the target scope
- final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId );
+ final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId, true);
//we may have to index we're indexing from source->target here
final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeFromSource( edge ) );
@@ -216,10 +214,8 @@ public class IndexServiceImpl implements IndexService {
return Optional.of(defaultProperties);
}
- //Steps to delete an IndexEdge.
- //1.Take the search edge given and search for all the edges in elasticsearch matching that search edge
- //2. Batch Delete all of those edges returned in the previous search.
- //TODO: optimize loops further.
+ // DO NOT USE THIS AS THE QUERY TO ES CAN CAUSE EXTREME LOAD
+ // TODO REMOVE THIS AND UPDATE THE TESTS TO NOT USE THIS METHOD
@Override
public Observable<IndexOperationMessage> deleteIndexEdge( final ApplicationScope applicationScope,
final Edge edge ) {
@@ -256,49 +252,31 @@ public class IndexServiceImpl implements IndexService {
return ObservableTimer.time( batches, addTimer );
}
-
@Override
- public Observable<IndexOperationMessage> deIndexEntity( final ApplicationScope applicationScope, final Id entityId,
- final UUID markedVersion,
- final List<UUID> allVersionsBeforeMarked ) {
-
- final EntityIndex ei = entityIndexFactory.
- createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
-
- // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code foe de-indexing
- // previously .timstamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data)
- final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
- CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId,
- Long.MAX_VALUE ) );
-
-
+ public Observable<IndexOperationMessage> deIndexEdge(final ApplicationScope applicationScope, final Edge edge,
+ final Id entityId, final UUID entityVersion){
- final EntityIndexBatch batch = ei.createBatch();
-
- // de-index each version of the entity before the marked version
- allVersionsBeforeMarked.forEach(version -> batch.deindex(searchEdgeFromSource, entityId, version));
-
-
- // for now, query the index to remove docs where the entity is source/target node and older than markedVersion
- // TODO: investigate getting this information from graph
- CandidateResults candidateResults = ei.getNodeDocsOlderThanMarked(entityId, markedVersion );
- candidateResults.forEach(candidateResult -> batch.deindex(candidateResult));
-
- return Observable.just(batch.build());
+ if (logger.isTraceEnabled()) {
+ logger.trace("deIndexEdge edge={} entityId={} entityVersion={}", edge.toString(), entityId.toString(), entityVersion.toString());
+ }
+ final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope));
+ final EntityIndexBatch entityBatch = ei.createBatch();
+ entityBatch.deindex(generateScopeFromSource( edge ), entityId, entityVersion);
+ return Observable.just(entityBatch.build());
}
+
@Override
public Observable<IndexOperationMessage> deIndexOldVersions(final ApplicationScope applicationScope,
final Id entityId,
- final List<UUID> versions,
- UUID markedVersion) {
+ final List<UUID> versions) {
final EntityIndex ei = entityIndexFactory.
createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
- // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code foe de-indexing
- // previously .timstamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data)
+ // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code for de-indexing
+ // previously .timsetamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data)
final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId,
Long.MAX_VALUE ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 4886b08..b1b7f75 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -122,12 +122,8 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
if (isDeleted) {
logger.info("Edge {} is deleted when seeking, deleting the edge", markedEdge);
- final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
-
- indexMessageObservable
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
@@ -136,18 +132,8 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
final Id sourceNodeId = markedEdge.getSourceNode();
logger.info("Edge {} has a deleted source node, deleting the entity for id {}", markedEdge, sourceNodeId);
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
-
- entityDeleteResults.getIndexObservable()
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
-
- Observable.merge(entityDeleteResults.getEntitiesDeleted(),
- entityDeleteResults.getCompactedNode())
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
- subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
@@ -156,19 +142,8 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
final Id targetNodeId = markedEdge.getTargetNode();
logger.info("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId);
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
-
- entityDeleteResults.getIndexObservable()
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
-
- Observable.merge(entityDeleteResults.getEntitiesDeleted(),
- entityDeleteResults.getCompactedNode())
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
- subscribe();
-
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
index e8a20dd..c75545e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
@@ -122,12 +122,8 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
if (isDeleted) {
logger.info("Edge {} is deleted when seeking, deleting the edge", markedEdge);
- final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
-
- indexMessageObservable
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
@@ -136,18 +132,8 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
final Id sourceNodeId = markedEdge.getSourceNode();
logger.info("Edge {} has a deleted source node, deleting the entity for id {}", markedEdge, sourceNodeId);
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
-
- entityDeleteResults.getIndexObservable()
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
-
- Observable.merge(entityDeleteResults.getEntitiesDeleted(),
- entityDeleteResults.getCompactedNode())
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
- subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
@@ -156,18 +142,8 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
final Id targetNodeId = markedEdge.getTargetNode();
logger.info("Edge {} has a deleted target node, deleting the entity for id {}", markedEdge, targetNodeId);
- final EventBuilderImpl.EntityDeleteResults
- entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
-
- entityDeleteResults.getIndexObservable()
- .compose(applyCollector(AsyncEventQueueType.DELETE))
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
- .subscribe();
-
- Observable.merge(entityDeleteResults.getEntitiesDeleted(),
- entityDeleteResults.getCompactedNode())
- .subscribeOn(rxTaskScheduler.getAsyncIOScheduler()).
- subscribe();
+ final IndexOperationMessage indexOperationMessage = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
+ asyncEventService.queueIndexOperationMessage(indexOperationMessage, AsyncEventQueueType.DELETE);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 900bda5..4c9d16c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -31,6 +31,8 @@ import org.apache.usergrid.persistence.index.query.tree.Operand;
import org.apache.usergrid.persistence.index.utils.ClassUtils;
import org.apache.usergrid.persistence.index.utils.ListUtils;
import org.apache.usergrid.persistence.index.utils.MapUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.Serializable;
@@ -43,6 +45,7 @@ import java.util.Map.Entry;
public class Query {
+ private static final Logger logger = LoggerFactory.getLogger(Query.class);
public enum Level {
IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
@@ -319,6 +322,13 @@ public class Query {
public static Query fromIdentifier( Object id ) {
+ if (id == null) {
+ throw new IllegalArgumentException("null identifier passed in");
+ }
+ Identifier objectIdentifier = Identifier.from(id);
+ if (objectIdentifier == null) {
+ throw new IllegalArgumentException("Supplied id results in null Identifier");
+ }
Query q = new Query();
q.addIdentifier( Identifier.from(id) );
return q;
@@ -409,6 +419,10 @@ public class Query {
}
for ( Identifier identifier : identifiers ) {
+ if (identifier == null) {
+ logger.error("containsUuidIdentifiersOnly(): identifier in identifiers list is null");
+ return false;
+ }
if ( !identifier.isUUID() ) {
return false;
}
@@ -635,6 +649,9 @@ public class Query {
if ( identifiers == null ) {
identifiers = new ArrayList<Identifier>();
}
+ if (identifier == null) {
+ throw new IllegalArgumentException("adding null identifier is not allowed");
+ }
identifiers.add( identifier );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java
index 3bfe460..68834b3 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservableIT.java
@@ -92,7 +92,7 @@ public class EdgesFromSourceObservableIT extends AbstractCoreIT {
final GraphManager gm = managerCache.getGraphManager( scope );
- edgesToTargetObservable.edgesToTarget( gm, target ).doOnNext( new Action1<Edge>() {
+ edgesToTargetObservable.edgesToTarget( gm, target, true).doOnNext(new Action1<Edge>() {
@Override
public void call( final Edge edge ) {
final String edgeType = edge.getType();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
index 9e84219..55b77c0 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservableIT.java
@@ -89,7 +89,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
final GraphManager gm = managerCache.getGraphManager( scope );
- edgesFromSourceObservable.edgesFromSourceDescending( gm, applicationId ).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceDescending( gm, applicationId, true).doOnNext(edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
@@ -118,7 +118,7 @@ public class EdgesToTargetObservableIT extends AbstractCoreIT {
//test connections
- edgesFromSourceObservable.edgesFromSourceDescending( gm, source ).doOnNext( edge -> {
+ edgesFromSourceObservable.edgesFromSourceDescending( gm, source, true).doOnNext(edge -> {
final String edgeType = edge.getType();
final Id target = edge.getTargetNode();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/core/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/project.properties b/stack/core/src/test/resources/project.properties
index 1a848bc..77a785a 100644
--- a/stack/core/src/test/resources/project.properties
+++ b/stack/core/src/test/resources/project.properties
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
target.directory=${project.build.directory}
-jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+jamm.path=-javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
index 000c633..b746c61 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphManager.java
@@ -93,7 +93,7 @@ public interface GraphManager extends CPManager {
* @param node The node to remove. This will apply a timestamp to apply the delete + compact operation. Any edges connected to this node with a timestamp
* <= the specified time on the mark will be removed from the graph
*/
- Observable<Id> compactNode( final Id node );
+ Observable<MarkedEdge> compactNode( final Id node );
/**
* Get all versions of this edge where versions <= max version
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
index 5fcdcb4..2fe40b1 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/GraphManagerImpl.java
@@ -262,24 +262,17 @@ public class GraphManagerImpl implements GraphManager {
@Override
- public Observable<Id> compactNode( final Id inputNode ) {
-
+ public Observable<MarkedEdge> compactNode( final Id inputNode ) {
final UUID startTime = UUIDGenerator.newTimeUUID();
-
- final Observable<Id> nodeObservable =
- Observable.just( inputNode ).map( node -> nodeSerialization.getMaxVersion( scope, node ) ).takeWhile(
- maxTimestamp -> maxTimestamp.isPresent() )
-
+ final Observable<MarkedEdge> nodeObservable =
+ Observable.just( inputNode )
+ .map( node -> nodeSerialization.getMaxVersion( scope, node ) )
+ //.doOnNext(maxTimestamp -> logger.info("compactNode maxTimestamp={}", maxTimestamp.toString()))
+ .takeWhile(maxTimestamp -> maxTimestamp.isPresent() )
//map our delete listener
- .flatMap( timestamp -> nodeDeleteListener.receive( scope, inputNode, startTime ) )
- //set to 0 if nothing is emitted
- .lastOrDefault( 0 )
- //log for posterity
- .doOnNext( count -> logger.trace( "Removed {} edges from node {}", count, inputNode ) )
- //return our id
- .map( count -> inputNode );
+ .flatMap( timestamp -> nodeDeleteListener.receive( scope, inputNode, startTime ) );
return ObservableTimer.time( nodeObservable, this.deleteNodeTimer );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
index 9392dbc..71d2f1d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
@@ -169,6 +169,15 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
return true;
}
+ @Override
+ public String toString(){
+ return "SimpleSearchByEdgeType{node="+node
+ +", type="+type
+ +", maxTimestamp="+maxTimestamp
+ +", order="+order
+ +", filterMarked="+filterMarked
+ +", last="+last+"}";
+ }
@Override
public int hashCode() {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
index 68569e5..3bcdc55 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListener.java
@@ -22,6 +22,8 @@ package org.apache.usergrid.persistence.graph.impl.stage;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
@@ -39,8 +41,8 @@ public interface NodeDeleteListener {
* @param node The node that was deleted
* @param timestamp The timestamp of the event
*
- * @return An observable that emits the total number of edges that have been removed with this node both as the
+ * @return An observable that emits the marked edges that have been removed with this node both as the
* target and source
*/
- Observable<Integer> receive( final ApplicationScope scope, final Id node, final UUID timestamp );
+ Observable<MarkedEdge> receive(final ApplicationScope scope, final Id node, final UUID timestamp );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index cd5b1a8..df4e5d5 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@ -95,53 +95,37 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
* @param node The node that was deleted
* @param timestamp The timestamp of the event
*
- * @return An observable that emits the total number of edges that have been removed with this node both as the
+ * @return An observable that emits the marked edges that have been removed with this node both as the
* target and source
*/
- public Observable<Integer> receive( final ApplicationScope scope, final Id node, final UUID timestamp ) {
+ public Observable<MarkedEdge> receive( final ApplicationScope scope, final Id node, final UUID timestamp ) {
return Observable.just( node )
//delete source and targets in parallel and merge them into a single observable
- .flatMap( new Func1<Id, Observable<Integer>>() {
- @Override
- public Observable<Integer> call( final Id node ) {
-
- final Optional<Long> maxVersion = nodeSerialization.getMaxVersion( scope, node );
-
- if (logger.isTraceEnabled()) {
- logger.trace("Node with id {} has max version of {}", node, maxVersion.orNull());
- }
-
-
- if ( !maxVersion.isPresent() ) {
- return Observable.empty();
- }
-
-
- //do all the delete, then when done, delete the node
- return doDeletes( node, scope, maxVersion.get(), timestamp ).count()
- //if nothing is ever emitted, emit 0 so that we know no operations took place.
- // Finally remove
- // the
- // target node in the mark
- .doOnCompleted( new Action0() {
- @Override
- public void call() {
- try {
- nodeSerialization.delete( scope, node, maxVersion.get()).execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to connect to casandra", e );
- }
- }
- } );
- }
- } ).defaultIfEmpty( 0 );
- }
+ .flatMap( id -> {
+ final Optional<Long> maxVersion = nodeSerialization.getMaxVersion( scope, node );
+
+ if (logger.isTraceEnabled()) {
+ logger.trace("Node with id {} has max version of {}", node, maxVersion.orNull());
+ }
+ if ( !maxVersion.isPresent() ) {
+ return Observable.empty();
+ }
+ // do all the edge deletes and then remove the marked node, return all edges just deleted
+ return
+ doDeletes( node, scope, maxVersion.get(), timestamp ).doOnCompleted( () -> {
+ try {
+ nodeSerialization.delete( scope, node, maxVersion.get()).execute();
+ } catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to connect to cassandra", e );
+ }
+ });
+ });
+ }
/**
* Do the deletes
*/
@@ -162,7 +146,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageSerialization.getEdgesToTarget(scope,
- new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent()));
+ new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent(), false));
}
}));
@@ -174,7 +158,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageSerialization.getEdgesFromSource(scope,
- new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent()));
+ new SimpleSearchByEdgeType(node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent(), false));
}
}));
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
index 78a1d4b..5577bd0 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/EdgesObservable.java
@@ -36,9 +36,10 @@ public interface EdgesObservable {
* Return an observable of all edges from a source
* @param gm
* @param sourceNode
+ * @param filterMarked
* @return
*/
- Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode );
+ Observable<Edge> edgesFromSourceDescending(final GraphManager gm, final Id sourceNode, boolean filterMarked);
/**
@@ -54,9 +55,10 @@ public interface EdgesObservable {
* Return an observable of all edges to a target
* @param gm
* @param targetNode
+ * @param filterMarked
* @return
*/
- Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode);
+ Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode, boolean filterMarked);
/**
* Return an observable of all edges from a source node. Ordered ascending, from the startTimestamp if specified
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
index 1f81864..e9e2b28 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationV2Impl.java
@@ -400,7 +400,6 @@ public class EdgeMetadataSerializationV2Impl implements EdgeMetadataSerializatio
ValidationUtils.validateApplicationScope( scope );
GraphValidation.validateSearchEdgeType( search );
-
final Id applicationId = scope.getApplication();
final Id searchNode = search.getNode();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index 20efe42..9e0998d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -35,7 +35,6 @@ import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
import rx.Observable;
-import rx.functions.Func1;
/**
@@ -55,7 +54,7 @@ public class EdgesObservableImpl implements EdgesObservable {
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesFromSourceDescending( final GraphManager gm, final Id sourceNode ) {
+ public Observable<Edge> edgesFromSourceDescending(final GraphManager gm, final Id sourceNode, boolean filterMarked) {
final Observable<String> edgeTypes =
gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( sourceNode, null, null ) );
@@ -67,7 +66,7 @@ public class EdgesObservableImpl implements EdgesObservable {
return gm.loadEdgesFromSource(
new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- Optional.<Edge>absent() ) );
+ Optional.<Edge>absent(), filterMarked ) );
} );
}
@@ -119,19 +118,16 @@ public class EdgesObservableImpl implements EdgesObservable {
* Get all edges from the source
*/
@Override
- public Observable<Edge> edgesToTarget( final GraphManager gm, final Id targetNode ) {
- final Observable<String> edgeTypes =
- gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( targetNode, null, null ) );
-
- return edgeTypes.flatMap( edgeType -> {
-
- if (logger.isTraceEnabled()) {
- logger.trace("Loading edges of edgeType {} to {}", edgeType, targetNode);
- }
+ public Observable<Edge> edgesToTarget(final GraphManager gm, final Id targetNode, boolean filterMarked) {
+ return gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( targetNode, null, null ) )
+ .flatMap( edgeType -> {
+ if (logger.isTraceEnabled()) {
+ logger.trace("Loading edges of edgeType {} to {}", edgeType, targetNode);
+ }
return gm.loadEdgesToTarget(
new SimpleSearchByEdgeType( targetNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- Optional.<Edge>absent() ) );
+ Optional.<Edge>absent(), filterMarked ) );
} );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
index 6a08d46..69dd43b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/TargetIdObservableImpl.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.persistence.graph.serialization.impl;
import com.google.inject.Inject;
-import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.serialization.EdgesObservable;
import org.apache.usergrid.persistence.graph.serialization.TargetIdObservable;
@@ -29,7 +28,6 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
-import rx.functions.Func1;
/**
* Emits the id of all nodes that are target nodes from the given source node
@@ -55,7 +53,7 @@ public class TargetIdObservableImpl implements TargetIdObservable {
public Observable<Id> getTargetNodes(final GraphManager gm, final Id sourceNode) {
//only search edge types that start with collections
- return edgesFromSourceObservable.edgesFromSourceDescending( gm, sourceNode ).map( edge -> {
+ return edgesFromSourceObservable.edgesFromSourceDescending( gm, sourceNode, true).map(edge -> {
final Id targetNode = edge.getTargetNode();
if (logger.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
index 8eccdbd..1d4331c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/EdgeDataMigrationImpl.java
@@ -89,7 +89,7 @@ public class EdgeDataMigrationImpl implements DataMigration {
final GraphManager gm = graphManagerFactory.createEdgeManager( graphNode.applicationScope );
//get edges from the source
- return edgesFromSourceObservable.edgesFromSourceDescending( gm, graphNode.entryNode ).buffer( 1000 )
+ return edgesFromSourceObservable.edgesFromSourceDescending( gm, graphNode.entryNode, true).buffer( 1000 )
.doOnNext( edges -> {
final MutationBatch batch = keyspace.prepareMutationBatch();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
index 438a978..80198de 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
@@ -135,7 +135,7 @@ public class NodeDeleteListenerTest {
UUID eventTime = UUIDGenerator.newTimeUUID();
- int count = deleteListener.receive( scope, sourceNode, eventTime ).toBlocking().last();
+ int count = deleteListener.receive( scope, sourceNode, eventTime ).count().toBlocking().last();
assertEquals( "Mark was not set, no delete should be executed", 0, count );
@@ -171,7 +171,7 @@ public class NodeDeleteListenerTest {
nodeSerialization.mark( scope, sourceNode, timestamp ).execute();
- int count = deleteListener.receive( scope, sourceNode, deleteEventTimestamp ).toBlocking().last();
+ int count = deleteListener.receive( scope, sourceNode, deleteEventTimestamp ).count().toBlocking().last();
assertEquals( 1, count );
@@ -256,7 +256,7 @@ public class NodeDeleteListenerTest {
nodeSerialization.mark( scope, targetNode, deleteBefore ).execute();
- int count = deleteListener.receive( scope, targetNode, UUIDGenerator.newTimeUUID() ).toBlocking().last();
+ int count = deleteListener.receive( scope, targetNode, UUIDGenerator.newTimeUUID() ).count().toBlocking().last();
assertEquals( 1, count );
@@ -366,7 +366,7 @@ public class NodeDeleteListenerTest {
nodeSerialization.mark( scope, toDelete, deleteVersion ).execute();
- int count = deleteListener.receive( scope, toDelete, UUIDGenerator.newTimeUUID() ).toBlocking().last();
+ int count = deleteListener.receive( scope, toDelete, UUIDGenerator.newTimeUUID() ).count().toBlocking().last();
assertEquals( edgeCount, count );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 14020a9..b444199 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -20,14 +20,12 @@
package org.apache.usergrid.persistence.index;
-import com.google.common.base.Optional;
import org.apache.usergrid.persistence.core.CPManager;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
import java.util.Map;
-import java.util.UUID;
/**
@@ -36,7 +34,7 @@ import java.util.UUID;
public interface EntityIndex extends CPManager {
- public static final int MAX_LIMIT = 1000;
+ int MAX_LIMIT = 1000;
/**
* Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
@@ -134,14 +132,6 @@ public interface EntityIndex extends CPManager {
CandidateResults getAllEdgeDocuments(final IndexEdge edge, final Id entityId);
/**
- * Returns all entity docs that match the entityId being the nodeId ( aka connections where entityId = sourceNode)
- *
- * @param entityId The entityId to match when searching
- * @param markedVersion The version that has been marked for deletion. All version before this one must be deleted.
- * @return
- */
- CandidateResults getNodeDocsOlderThanMarked(final Id entityId, final UUID markedVersion);
- /**
* delete all application records
*
* @return
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index cb695d5..f4fae2b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -34,7 +34,6 @@ import org.apache.usergrid.persistence.core.migration.data.VersionedData;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.StringUtils;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.ElasticSearchQueryBuilder.SearchRequestBuilderStrategyV2;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
@@ -584,68 +583,6 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
}
- @Override
- public CandidateResults getNodeDocsOlderThanMarked(final Id entityId, final UUID markedVersion ) {
-
- // TODO: investigate if functionality via iterator so a caller can page the deletion until all is gone
-
- Preconditions.checkNotNull( entityId, "entityId cannot be null" );
- Preconditions.checkNotNull(markedVersion, "markedVersion cannot be null");
- ValidationUtils.verifyVersion(markedVersion);
-
- SearchResponse searchResponse;
- List<CandidateResult> candidates = new ArrayList<>();
-
- final long markedTimestamp = markedVersion.timestamp();
-
- // never let this fetch more than 100 to save memory
- final int searchLimit = Math.min(100, indexFig.getVersionQueryLimit());
-
- // this query will find all the documents where this entity is a source/target node
- final QueryBuilder nodeQuery = QueryBuilders
- .termQuery(IndexingUtils.EDGE_NODE_ID_FIELDNAME, IndexingUtils.nodeId(entityId));
-
- final SearchRequestBuilder srb = searchRequestBuilderStrategyV2.getBuilder()
- .addSort(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME, SortOrder.ASC);
-
- try {
-
- long queryTimestamp = 0L;
-
- QueryBuilder timestampQuery = QueryBuilders
- .rangeQuery(IndexingUtils.EDGE_TIMESTAMP_FIELDNAME)
- .gte(queryTimestamp)
- .lt(markedTimestamp);
-
- QueryBuilder finalQuery = QueryBuilders.constantScoreQuery(
- QueryBuilders
- .boolQuery()
- .must(timestampQuery)
- .must(nodeQuery)
- );
-
-
- searchResponse = srb
- .setQuery(finalQuery)
- .setSize(searchLimit)
- .execute()
- .actionGet();
-
-
- candidates = aggregateScrollResults(candidates, searchResponse, markedVersion);
-
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t.getMessage() );
- failureMonitor.fail( "Unable to execute batch", t );
- throw t;
- }
- failureMonitor.success();
-
- return new CandidateResults( candidates, Collections.EMPTY_SET);
- }
-
-
/**
* Completely delete an index.
*/
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
index 84a28f0..70d2284 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Identifier.java
@@ -34,6 +34,7 @@ public class Identifier implements Serializable {
public static final String UUID_REX =
"[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}";
public static final String EMAIL_REX = "[a-zA-Z0-9._%'+\\-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}";
+ public static final String NAME_REX = "[a-zA-Z0-9_\\-./'+ ]*";
public enum Type {
UUID, NAME, EMAIL
@@ -46,7 +47,7 @@ public class Identifier implements Serializable {
static Pattern emailRegEx = Pattern.compile( EMAIL_REX );
// "Pattern nameRegEx" below used to be [a-zA-Z0-9_\\-./], changed it to contain a 'space' to a
// ddress https://issues.apache.org/jira/browse/USERGRID-94
- static Pattern nameRegEx = Pattern.compile( "[a-zA-Z0-9_\\-./'+ ]*" );
+ static Pattern nameRegEx = Pattern.compile( NAME_REX );
private Identifier( Type type, Object value ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 513c894..c98c72d 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -661,9 +661,9 @@
</dependency>
<dependency>
- <groupId>com.github.stephenc</groupId>
+ <groupId>com.github.jbellis</groupId>
<artifactId>jamm</artifactId>
- <version>0.2.5</version>
+ <version>0.3.1</version>
</dependency>
<!-- Third Party Non-Commercial Dependencies -->
@@ -1311,7 +1311,7 @@
<useSystemClassLoader>false</useSystemClassLoader>
<testFailureIgnore>false</testFailureIgnore>
<argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin}
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
${ug.argline}
</argLine>
<systemPropertyVariables>
@@ -1548,7 +1548,7 @@
<version>${surefire.plugin.version}</version>
<configuration>
<argLine>
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
${ug.argline}
-javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec
</argLine>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/rest/pom.xml
----------------------------------------------------------------------
diff --git a/stack/rest/pom.xml b/stack/rest/pom.xml
index 9bb83a6..e7aa68a 100644
--- a/stack/rest/pom.xml
+++ b/stack/rest/pom.xml
@@ -93,7 +93,7 @@
<argLine>-Dwebapp.directory=${basedir}/src/main/webapp
-Dtest.barrier.timestamp=${maven.build.timestamp} -Dtest.clean.storage=true -Xmx${ug.heapmax}
-Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
-Djava.util.logging.config.file=${basedir}/src/test/resources/logging.properties ${ug.argline}
</argLine>
<includes>
@@ -460,7 +460,7 @@
-Dwebapp.directory=${basedir}/src/main/webapp
-Dtest.barrier.timestamp=${maven.build.timestamp} -Dtest.clean.storage=true
-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
-Djava.util.logging.config.file=${basedir}/src/test/resources/logging.properties
${ug.argline}
</argLine>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/rest/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/project.properties b/stack/rest/src/test/resources/project.properties
index 94ef3bd..20a38f6 100644
--- a/stack/rest/src/test/resources/project.properties
+++ b/stack/rest/src/test/resources/project.properties
@@ -15,4 +15,4 @@
# limitations under the License.
target.directory=${project.build.directory}
-jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+jamm.path=-javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index b1df1b4..29fa311 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -102,7 +102,7 @@
<useSystemClassLoader>false</useSystemClassLoader>
<argLine>-Dtest.barrier.timestamp=${maven.build.timestamp} -Dtest.clean.storage=true
-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
${ug.argline} -Dlog4j.configuration=file:${basedir}/src/test/resources/log4j.properties
</argLine>
<includes>
@@ -499,7 +499,7 @@
<configuration>
<argLine>-Dtest.barrier.timestamp=${maven.build.timestamp} -Dtest.clean.storage=true
-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8
- -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+ -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
${ug.argline}
-javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec
</argLine>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/services/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/project.properties b/stack/services/src/test/resources/project.properties
index d38e878..03736c0 100644
--- a/stack/services/src/test/resources/project.properties
+++ b/stack/services/src/test/resources/project.properties
@@ -16,4 +16,4 @@
target.directory=${project.build.directory}
resources.dir=${project.build.directory}
-jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+jamm.path=-javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/stack/test-utils/pom.xml b/stack/test-utils/pom.xml
index bbcf1ff..f99d43a 100644
--- a/stack/test-utils/pom.xml
+++ b/stack/test-utils/pom.xml
@@ -59,7 +59,7 @@
<threadCount>${usergrid.it.threads}</threadCount>
<threadCountClasses></threadCountClasses>
<reuseForks>true</reuseForks>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.0/jamm-0.3.0.jar ${ug.argline}</argLine>
<includes>
<include>**/CassandraResourceITSuite.java</include>
</includes>
@@ -290,7 +290,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>${surefire.plugin.version}</version>
<configuration>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.0/jamm-0.3.0.jar ${ug.argline}</argLine>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/test-utils/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/test/resources/project.properties b/stack/test-utils/src/test/resources/project.properties
index cd5b819..0bc9bb7 100644
--- a/stack/test-utils/src/test/resources/project.properties
+++ b/stack/test-utils/src/test/resources/project.properties
@@ -14,4 +14,4 @@
# limitations under the License.
target.directory=${project.build.directory}
-jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar
+jamm.path=-javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.1/jamm-0.3.1.jar
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/tools/pom.xml
----------------------------------------------------------------------
diff --git a/stack/tools/pom.xml b/stack/tools/pom.xml
index cbd2c1e..b34b068 100644
--- a/stack/tools/pom.xml
+++ b/stack/tools/pom.xml
@@ -61,7 +61,7 @@
<storage-config>${basedir}/src/test/conf</storage-config>
</systemPropertyVariables>
<forkMode>always</forkMode>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.0/jamm-0.3.0.jar ${ug.argline}</argLine>
</configuration>
</plugin>
[03/27] usergrid git commit: add a little more wait time
Posted by mr...@apache.org.
add a little more wait time
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/36e28a87
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/36e28a87
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/36e28a87
Branch: refs/heads/expose-reindex
Commit: 36e28a8704bd129be9c78b16fb4f0b77e97381df
Parents: 7d9cc89
Author: Mike Dunker <md...@google.com>
Authored: Wed Aug 16 22:30:34 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Wed Aug 16 22:30:34 2017 -0700
----------------------------------------------------------------------
.../apache/usergrid/rest/applications/queries/QueryTestBase.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/36e28a87/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
index 0adafef..1eec9ae 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/QueryTestBase.java
@@ -65,7 +65,7 @@ public class QueryTestBase extends AbstractRestIT {
logger.info(entities[i].entrySet().toString());
}
//refresh the index so that they are immediately searchable
- this.waitForQueueDrainAndRefreshIndex();
+ this.waitForQueueDrainAndRefreshIndex(1000);
return entities;
}
[26/27] usergrid git commit: code review changes
Posted by mr...@apache.org.
code review changes
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4a659106
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4a659106
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4a659106
Branch: refs/heads/expose-reindex
Commit: 4a659106eadefc83cce93f47d6b5090f65b8c1e3
Parents: 9a15108
Author: Peter Johnson <pj...@apigee.com>
Authored: Wed Sep 27 15:10:56 2017 -0700
Committer: Peter Johnson <pj...@apigee.com>
Committed: Wed Sep 27 15:10:56 2017 -0700
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpEntityManager.java | 4 ++--
.../corepersistence/asyncevents/EventBuilderImpl.java | 1 -
.../persistence/queue/impl/SNSQueueManagerImpl.java | 11 +++++++----
3 files changed, 9 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4a659106/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 1dc4a89..3f3794e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2854,7 +2854,7 @@ public class CpEntityManager implements EntityManager {
// add to and index in collection of the application
if ( !is_application) {
- updateIndexForEniity(eType, entity, timestamp);
+ updateIndexForEntity(eType, entity, timestamp);
}
//write to our types map
@@ -2864,7 +2864,7 @@ public class CpEntityManager implements EntityManager {
return entity;
}
- private <A extends Entity> void updateIndexForEniity(String eType, A entity, long timestamp) throws Exception {
+ private <A extends Entity> void updateIndexForEntity(String eType, A entity, long timestamp) throws Exception {
String collectionName = Schema.defaultCollectionName( eType );
CpRelationManager cpr = ( CpRelationManager ) getRelationManager( getApplication() );
cpr.addToCollection( collectionName, entity );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4a659106/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 203d32a..1eb5e03 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.UUID;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4a659106/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index 775a64b..bc9be57 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -84,6 +84,8 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
private static final int MIN_VISIBILITY_TIMEOUT = 1; //seconds
private static final String DEAD_LETTER_QUEUE_SUFFIX = "_dead";
+ private static final String FAILED_TO_SEND_MESSAGE = "FAILED INDEX REQUEST: Failed to send message to SNS Queue, sending asynchronously. Message:[{}] URL:[{}] Error:[{}]";
+
static {
/**
@@ -573,7 +575,7 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
}
} catch (Exception e) {
if (logger.isErrorEnabled()) {
- logger.error("Failed to send this message {} to SNS queue at {}", stringBody, topicArn);
+ logger.error(FAILED_TO_SEND_MESSAGE, stringBody, topicArn, e);
}
sendMessageToAllRegionsAsync(body);
}
@@ -591,7 +593,7 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
final String stringBody = toString( body );
- String topicArn = getWriteTopicArn();
+ final String topicArn = getWriteTopicArn();
if ( logger.isTraceEnabled() ) {
logger.trace( "Publishing Message...{} to arn: {}", stringBody, topicArn );
@@ -603,6 +605,7 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
@Override
public void onError( Exception e ) {
logger.error( "Error publishing message... {}", e );
+ logger.error(FAILED_TO_SEND_MESSAGE, stringBody, topicArn, e);
}
@@ -713,7 +716,7 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
url);
}
} catch (Exception e) {
- logger.error("Failed to send this message {}. To this address {}. Error was ", messageRequest.getMessageBody(), url, e);
+ logger.error(FAILED_TO_SEND_MESSAGE, messageRequest.getMessageBody(), url, e);
sendMessageToLocalRegionAsync(body);
}
@@ -740,7 +743,7 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
@Override
public void onError( final Exception e ) {
- logger.error("Failed to send this message {}. To this address {}. Error was ", stringBody, url, e);
+ logger.error(FAILED_TO_SEND_MESSAGE, stringBody, url, e);
}
[25/27] usergrid git commit: Allow submission to SNS/SQS via sync
client
Posted by mr...@apache.org.
Allow submission to SNS/SQS via sync client
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9a151089
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9a151089
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9a151089
Branch: refs/heads/expose-reindex
Commit: 9a15108924b29997c9ca440b467ab8006a8f8acb
Parents: 173be8a
Author: Peter Johnson <pj...@apigee.com>
Authored: Wed Sep 27 08:19:20 2017 -0700
Committer: Peter Johnson <pj...@apigee.com>
Committed: Wed Sep 27 08:19:20 2017 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 75 +++++++------
.../corepersistence/CpRelationManager.java | 52 ++++-----
.../asyncevents/AsyncEventService.java | 6 +-
.../asyncevents/AsyncEventServiceImpl.java | 44 +++++---
.../asyncevents/EventBuilderImpl.java | 2 +
.../corepersistence/util/CpCollectionUtils.java | 104 +++++++++++++++++
.../index/AsyncIndexServiceTest.java | 2 +-
.../persistence/queue/LegacyQueueFig.java | 5 +
.../persistence/queue/LegacyQueueManager.java | 11 +-
.../persistence/queue/LocalQueueManager.java | 11 +-
.../queue/impl/QakkaQueueManager.java | 12 +-
.../queue/impl/SNSQueueManagerImpl.java | 112 +++++++++++++++++--
.../queue/LegacyQueueManagerTest.java | 2 +-
.../impl/ApplicationQueueManagerImpl.java | 2 +-
.../services/queues/ImportQueueManager.java | 8 +-
15 files changed, 341 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 68c4ef0..1dc4a89 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -34,6 +34,7 @@ import org.apache.usergrid.corepersistence.index.CollectionSettingsFactory;
import org.apache.usergrid.corepersistence.index.CollectionSettingsScopeImpl;
import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.util.CpCollectionUtils;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.mq.QueueManager;
@@ -161,7 +162,6 @@ public class CpEntityManager implements EntityManager {
public QueueManagerFactory queueManagerFactory;
-
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -521,6 +521,10 @@ public class CpEntityManager implements EntityManager {
cpEntity = CpEntityMapUtils.fromMap( cpEntity, entity.getProperties(), entity.getType(), true );
+ String entityType = cpEntity.getId().getType();
+ boolean skipIndexingForType = skipIndexingForType(entityType);
+ Boolean asyncIndex = asyncIndexingForType(entityType);
+
try {
String region = lookupAuthoritativeRegionForType( entity.getType() );
@@ -546,38 +550,32 @@ public class CpEntityManager implements EntityManager {
handleWriteUniqueVerifyException( entity, wuve );
}
- if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
+ if (!skipIndexingForType) {
+ indexEntity(cpEntity, asyncIndex);
+ deIndexOldVersionsOfEntity(cpEntity);
+ }
+ }
- // queue an event to update the new entity
- indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+ private void indexEntity(org.apache.usergrid.persistence.model.entity.Entity cpEntity, Boolean async) {
+ // queue an event to update the new entity
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 , async);
+ }
- // queue up an event to clean-up older versions than this one from the index
- if (entityManagerFig.getDeindexOnUpdate()) {
- indexService.queueDeIndexOldVersion( applicationScope, cpEntity.getId(), cpEntity.getVersion());
- }
+ private void deIndexOldVersionsOfEntity(org.apache.usergrid.persistence.model.entity.Entity cpEntity) {
+ // queue up an event to clean-up older versions than this one from the index
+ if (entityManagerFig.getDeindexOnUpdate()) {
+ indexService.queueDeIndexOldVersion( applicationScope, cpEntity.getId(), cpEntity.getVersion());
}
}
- private boolean skipIndexingForType( String type ) {
-
- boolean skipIndexing = false;
- String collectionName = Schema.defaultCollectionName( type );
+ private Boolean asyncIndexingForType( String type ) {
+ return CpCollectionUtils.asyncIndexingForType(collectionSettingsFactory, applicationId, type);
- CollectionSettings collectionSettings = collectionSettingsFactory
- .getInstance( new CollectionSettingsScopeImpl(getAppIdObject(), collectionName) );
- Optional<Map<String, Object>> existingSettings =
- collectionSettings.getCollectionSettings( collectionName );
-
- if ( existingSettings.isPresent()) {
- Map jsonMapData = existingSettings.get();
- Object fields = jsonMapData.get("fields");
- if ( fields != null && "none".equalsIgnoreCase( fields.toString() ) ) {
- skipIndexing = true;
- }
- }
+ }
- return skipIndexing;
+ private boolean skipIndexingForType( String type ) {
+ return CpCollectionUtils.skipIndexingForType(collectionSettingsFactory, applicationId, type);
}
@@ -1153,7 +1151,7 @@ public class CpEntityManager implements EntityManager {
//Adding graphite metrics
if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
- indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 , null);
}
}
@@ -1813,9 +1811,10 @@ public class CpEntityManager implements EntityManager {
updatedSettings.put( "lastReindexed", 0 );
}
- // if fields specified, then put in settings
- if ( newSettings.get("fields") != null ) {
- updatedSettings.put("fields", newSettings.get("fields"));
+ for (String validName : CpCollectionUtils.getValidSettings()) {
+ if (newSettings.containsKey(validName)) {
+ updatedSettings.put(validName, newSettings.get(validName));
+ }
}
// if region specified
@@ -2854,14 +2853,8 @@ public class CpEntityManager implements EntityManager {
entity.setProperties( cpEntity );
// add to and index in collection of the application
- if ( !is_application ) {
-
- String collectionName = Schema.defaultCollectionName( eType );
- CpRelationManager cpr = ( CpRelationManager ) getRelationManager( getApplication() );
- cpr.addToCollection( collectionName, entity );
-
- // Invoke counters
- incrementEntityCollection( collectionName, timestamp );
+ if ( !is_application) {
+ updateIndexForEniity(eType, entity, timestamp);
}
//write to our types map
@@ -2871,6 +2864,14 @@ public class CpEntityManager implements EntityManager {
return entity;
}
+ private <A extends Entity> void updateIndexForEniity(String eType, A entity, long timestamp) throws Exception {
+ String collectionName = Schema.defaultCollectionName( eType );
+ CpRelationManager cpr = ( CpRelationManager ) getRelationManager( getApplication() );
+ cpr.addToCollection( collectionName, entity );
+
+ // Invoke counters
+ incrementEntityCollection( collectionName, timestamp );
+ }
private void incrementEntityCollection( String collection_name, long cassandraTimestamp ) {
try {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index c02ca7d..06f06ad 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.corepersistence.service.CollectionSearch;
import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionSearch;
import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.util.CpCollectionUtils;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.*;
@@ -349,6 +350,7 @@ public class CpRelationManager implements RelationManager {
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
org.apache.usergrid.persistence.model.entity.Entity memberEntity = ( ( CpEntityManager ) em ).load( entityId );
+ Id memberEntityId = memberEntity.getId();
// don't fetch entity if we've already got one
final Entity itemEntity;
@@ -364,7 +366,7 @@ public class CpRelationManager implements RelationManager {
}
- if ( memberEntity == null ) {
+ if ( memberEntityId == null ) {
throw new RuntimeException(
"Unable to load entity uuid=" + itemRef.getUuid() + " type=" + itemRef.getType() );
}
@@ -376,7 +378,7 @@ public class CpRelationManager implements RelationManager {
// create graph edge connection from head entity to member entity
- final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntity.getId() );
+ final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntityId );
final String linkedCollection = collection.getLinkedCollection();
GraphManager gm = managerCache.getGraphManager( applicationScope );
@@ -387,21 +389,24 @@ public class CpRelationManager implements RelationManager {
}
} ).filter( writtenEdge -> linkedCollection != null ).flatMap( writtenEdge -> {
final String pluralType = InflectionUtils.pluralize( cpHeadEntity.getId().getType() );
- final Edge reverseEdge = createCollectionEdge( memberEntity.getId(), pluralType, cpHeadEntity.getId() );
+ final Edge reverseEdge = createCollectionEdge( memberEntityId, pluralType, cpHeadEntity.getId() );
//reverse
return gm.writeEdge( reverseEdge ).doOnNext( reverseEdgeWritten -> {
- if ( !skipIndexingForType( cpHeadEntity.getId().getType() ) ) {
-
- indexService.queueNewEdge(applicationScope, cpHeadEntity, reverseEdge);
+ String entityType = cpHeadEntity.getId().getType();
+ if ( !skipIndexingForType( entityType) ) {
+ Boolean async = asyncIndexingForType(entityType);
+ indexService.queueNewEdge(applicationScope, cpHeadEntity.getId(), reverseEdge, async);
}
} );
} ).doOnCompleted( () -> {
- if ( !skipIndexingForType( memberEntity.getId().getType() ) ) {
- indexService.queueNewEdge(applicationScope, memberEntity, edge);
+ String entityType = memberEntity.getId().getType();
+ if ( !skipIndexingForType( entityType ) ) {
+ Boolean async = asyncIndexingForType(entityType);
+ indexService.queueNewEdge(applicationScope, memberEntityId, edge, async);
}
@@ -731,9 +736,10 @@ public class CpRelationManager implements RelationManager {
gm.writeEdge(edge).toBlocking().lastOrDefault(null); //throw an exception if this fails
- if ( !skipIndexingForType( targetEntity.getId().getType() ) ) {
-
- indexService.queueNewEdge(applicationScope, targetEntity, edge);
+ String entityType = targetEntity.getId().getType();
+ if ( !skipIndexingForType( entityType ) ) {
+ Boolean async = asyncIndexingForType(entityType);
+ indexService.queueNewEdge(applicationScope, targetEntity.getId(), edge, async);
}
// remove any duplicate edges (keeps the duplicate edge with same timestamp)
@@ -1094,27 +1100,13 @@ public class CpRelationManager implements RelationManager {
}
- private boolean skipIndexingForType( String type ) {
-
- boolean skipIndexing = false;
-
- String collectionName = Schema.defaultCollectionName( type );
+ private Boolean asyncIndexingForType( String type ) {
+ return CpCollectionUtils.asyncIndexingForType(collectionSettingsFactory, applicationId, type);
- CollectionSettings collectionSettings =
- collectionSettingsFactory.
- getInstance( new CollectionSettingsScopeImpl(new SimpleId( applicationId, TYPE_APPLICATION ), collectionName ) );
- Optional<Map<String, Object>> collectionIndexingSchema =
- collectionSettings.getCollectionSettings( collectionName );
-
- if ( collectionIndexingSchema.isPresent()) {
- Map jsonMapData = collectionIndexingSchema.get();
- final Object fields = jsonMapData.get( "fields" );
- if ( fields != null && fields instanceof String && "none".equalsIgnoreCase( fields.toString())) {
- skipIndexing = true;
- }
- }
+ }
- return skipIndexing;
+ private boolean skipIndexingForType( String type ) {
+ return CpCollectionUtils.skipIndexingForType(collectionSettingsFactory, applicationId, type);
}
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index 04eaf4c..1ddbac4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -53,7 +53,7 @@ public interface AsyncEventService extends ReIndexAction, CollectionDeleteAction
* @param entity The entity to index. Should be fired when an entity is updated
* @param updatedAfter
*/
- void queueEntityIndexUpdate(final ApplicationScope applicationScope, final Entity entity, long updatedAfter);
+ void queueEntityIndexUpdate(final ApplicationScope applicationScope, final Entity entity, long updatedAfter, Boolean async);
/**
@@ -63,10 +63,10 @@ public interface AsyncEventService extends ReIndexAction, CollectionDeleteAction
* TODO: We shouldn't take an entity here, only the id. It doesn't make sense in a distributed context
*
* @param applicationScope
- * @param entity
+ * @param entityId
* @param newEdge
*/
- void queueNewEdge(final ApplicationScope applicationScope, final Entity entity, final Edge newEdge);
+ void queueNewEdge(final ApplicationScope applicationScope, final Id entityId, final Edge newEdge, Boolean async);
/**
* Queue the deletion of an edge
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 3d06cae..3e67110 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -276,15 +276,26 @@ public class AsyncEventServiceImpl implements AsyncEventService {
* Offer the EntityIdScope to SQS
*/
private void offer(final Serializable operation) {
- offer(operation, AsyncEventQueueType.REGULAR);
+ offer(operation, AsyncEventQueueType.REGULAR, null);
}
- private void offer(final Serializable operation, AsyncEventQueueType queueType) {
+ /**
+ * Offer the EntityIdScope to SQS
+ */
+ private void offer(final Serializable operation, Boolean async) {
+ offer(operation, AsyncEventQueueType.REGULAR, async);
+ }
+
+ /**
+ * Offer the EntityIdScope to SQS
+ */
+ private void offer(final Serializable operation, AsyncEventQueueType queueType, Boolean async) {
final Timer.Context timer = this.writeTimer.time();
try {
//signal to SQS
- getQueue(queueType).sendMessageToLocalRegion(operation);
+ getQueue(queueType).sendMessageToLocalRegion(operation, async);
+
} catch (IOException e) {
throw new RuntimeException("Unable to queue message", e);
} finally {
@@ -299,7 +310,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
try {
//signal to SQS
- getQueue(queueType).sendMessageToAllRegions(operation);
+ getQueue(queueType).sendMessageToAllRegions(operation,null);
+
}
catch ( IOException e ) {
throw new RuntimeException( "Unable to queue message", e );
@@ -530,7 +542,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
@Override
public void queueEntityIndexUpdate(final ApplicationScope applicationScope,
- final Entity entity, long updatedAfter) {
+ final Entity entity, long updatedAfter, Boolean async) {
if (logger.isTraceEnabled()) {
@@ -538,13 +550,18 @@ public class AsyncEventServiceImpl implements AsyncEventService {
entity.getId().getUuid(), entity.getId().getType());
}
- offer(new EntityIndexEvent(queueFig.getPrimaryRegion(),
- new EntityIdScope(applicationScope, entity.getId()), updatedAfter));
+
+ EntityIndexEvent event = new EntityIndexEvent(queueFig.getPrimaryRegion(),
+ new EntityIdScope(applicationScope, entity.getId()),
+ updatedAfter);
+
+ offer(event, async);
}
private IndexOperationMessage handleEntityIndexUpdate(final LegacyQueueMessage message) {
+
Preconditions.checkNotNull( message, "Queue Message cannot be null for handleEntityIndexUpdate" );
final AsyncEvent event = ( AsyncEvent ) message.getBody();
@@ -574,15 +591,16 @@ public class AsyncEventServiceImpl implements AsyncEventService {
@Override
public void queueNewEdge(final ApplicationScope applicationScope,
- final Entity entity,
- final Edge newEdge) {
+ final Id entityId,
+ final Edge newEdge,
+ Boolean async) {
if (logger.isTraceEnabled()) {
logger.trace("Offering EdgeIndexEvent for edge type {} entity {}:{}",
- newEdge.getType(), entity.getId().getUuid(), entity.getId().getType());
+ newEdge.getType(), entityId.getUuid(), entityId.getType());
}
- offer( new EdgeIndexEvent( queueFig.getPrimaryRegion(), applicationScope, entity.getId(), newEdge ));
+ offer( new EdgeIndexEvent( queueFig.getPrimaryRegion(), applicationScope, entityId, newEdge ), async);
}
@@ -620,7 +638,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
- offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ), AsyncEventQueueType.DELETE );
+ offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ), AsyncEventQueueType.DELETE , null);
}
private IndexOperationMessage handleEdgeDelete(final LegacyQueueMessage message) {
@@ -824,7 +842,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
offer( new EntityDeleteEvent(queueFig.getPrimaryRegion(), new EntityIdScope( applicationScope, entityId ) ),
- AsyncEventQueueType.DELETE );
+ AsyncEventQueueType.DELETE , null);
}
private IndexOperationMessage handleEntityDelete(final LegacyQueueMessage message) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 7c72b72..203d32a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.UUID;
@@ -303,6 +304,7 @@ public class EventBuilderImpl implements EventBuilder {
return indexService.deIndexOldVersions( applicationScope, entityId,
getVersionsOlderThanOrEqualToMarked(ecm, entityId, markedVersion));
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpCollectionUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpCollectionUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpCollectionUtils.java
new file mode 100644
index 0000000..cef6d12
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpCollectionUtils.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.corepersistence.util;
+
+import com.google.common.base.Optional;
+import org.apache.usergrid.corepersistence.index.CollectionSettings;
+import org.apache.usergrid.corepersistence.index.CollectionSettingsFactory;
+import org.apache.usergrid.corepersistence.index.CollectionSettingsScopeImpl;
+
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import java.util.*;
+
+import static org.apache.usergrid.persistence.Schema.*;
+
+
+/**
+ *
+ * Helper methods to manage the Collection setting properties
+ *
+ */
+public class CpCollectionUtils {
+
+
+ public static final String SETTING_FIELDS = "fields";
+ public static final String SETTING_QUEUE_INDEX = "queueIndex";
+
+ private static Set<String> VALID_SETTING_NAMES = new HashSet<>();
+
+ static {
+ VALID_SETTING_NAMES.add(SETTING_FIELDS);
+ VALID_SETTING_NAMES.add(SETTING_QUEUE_INDEX);
+ }
+
+ public static Set<String> getValidSettings() {
+ return VALID_SETTING_NAMES;
+ }
+
+ public static Boolean asyncIndexingForType(CollectionSettingsFactory collectionSettingsFactory, UUID applicationId, String type ) {
+
+ String indexing = getFieldForType(applicationId, collectionSettingsFactory, type, SETTING_QUEUE_INDEX);
+ if ("async".equals(indexing)) {
+ return Boolean.TRUE;
+ }
+ if ("sync".equals(indexing)) {
+ return Boolean.FALSE;
+ }
+ return null;
+ }
+
+ public static boolean skipIndexingForType(CollectionSettingsFactory collectionSettingsFactory, UUID applicationId, String type ) {
+
+ String fields = getFieldForType(applicationId, collectionSettingsFactory, type, SETTING_FIELDS);
+ boolean skipIndexing = false;
+ if ( fields != null && fields instanceof String && "none".equalsIgnoreCase( fields.toString())) {
+ skipIndexing = true;
+ }
+
+ return skipIndexing;
+ }
+
+ // these same methods are in CpEntityManager must refactor
+ private static String getFieldForType(UUID applicationId, CollectionSettingsFactory collectionSettingsFactory,
+ String type, String keyName ) {
+
+ String collectionName = Schema.defaultCollectionName( type );
+
+ CollectionSettings collectionSettings = collectionSettingsFactory
+ .getInstance( new CollectionSettingsScopeImpl(getAppIdObject(applicationId), collectionName) );
+ Optional<Map<String, Object>> existingSettings =
+ collectionSettings.getCollectionSettings( collectionName );
+
+ if ( existingSettings.isPresent()) {
+ Map jsonMapData = existingSettings.get();
+ Object value = jsonMapData.get(keyName);
+ if ( value != null) {
+ return value.toString();
+ }
+ }
+ return null;
+ }
+
+ private static Id getAppIdObject(UUID applicationId){
+ return new SimpleId( applicationId, TYPE_APPLICATION );
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
index cecc3b2..766e2b2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
@@ -144,7 +144,7 @@ public abstract class AsyncIndexServiceTest {
//queue up processing
- asyncEventService.queueEntityIndexUpdate( applicationScope, testEntity, 0);
+ asyncEventService.queueEntityIndexUpdate( applicationScope, testEntity, 0, null);
final EntityIndex EntityIndex =
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
index 0ebcc7b..f19bede 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
@@ -109,4 +109,9 @@ public interface LegacyQueueFig extends GuicyFig {
@Key("usergrid.queue.map.message.timeout")
@Default("900000") // 15 minutes
int getMapMessageTimeout();
+
+ @Key("usergrid.queue.is.async")
+ @Default("true")
+ boolean isAsyncQueue();
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java
index f153610..148cb5d 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java
@@ -66,6 +66,13 @@ public interface LegacyQueueManager {
* @param bodies body objects must be serializable
* @throws IOException
*/
+ void sendMessagesAsync(List bodies) throws IOException;
+
+ /**
+ * send messages to queue
+ * @param bodies body objects must be serializable
+ * @throws IOException
+ */
void sendMessages(List bodies) throws IOException;
/**
@@ -81,13 +88,13 @@ public interface LegacyQueueManager {
* @param body
* @throws IOException
*/
- <T extends Serializable> void sendMessageToLocalRegion(T body)throws IOException;
+ <T extends Serializable> void sendMessageToLocalRegion(T body, Boolean async)throws IOException;
/**
* Send a messae to the topic to be sent to other queues
* @param body
*/
- <T extends Serializable> void sendMessageToAllRegions(T body) throws IOException;
+ <T extends Serializable> void sendMessageToAllRegions(T body, Boolean async) throws IOException;
/**
* purge messages
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java
index cbba0b1..1f07b58 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java
@@ -79,6 +79,11 @@ public class LocalQueueManager implements LegacyQueueManager {
}
@Override
+ public void sendMessagesAsync(List bodies) throws IOException {
+ sendMessages(bodies);
+ }
+
+ @Override
public void sendMessages(List bodies) throws IOException {
for(Object body : bodies){
String uuid = UUID.randomUUID().toString();
@@ -108,7 +113,7 @@ public class LocalQueueManager implements LegacyQueueManager {
@Override
- public <T extends Serializable> void sendMessageToLocalRegion(final T body ) throws IOException {
+ public <T extends Serializable> void sendMessageToLocalRegion(final T body, Boolean async) throws IOException {
String uuid = UUID.randomUUID().toString();
try {
queue.offer(new LegacyQueueMessage(uuid, "handle_" + uuid, body, "put type here"),5000,TimeUnit.MILLISECONDS);
@@ -120,8 +125,8 @@ public class LocalQueueManager implements LegacyQueueManager {
@Override
- public <T extends Serializable> void sendMessageToAllRegions(final T body ) throws IOException {
- sendMessageToLocalRegion( body );
+ public <T extends Serializable> void sendMessageToAllRegions(final T body, Boolean async) throws IOException {
+ sendMessageToLocalRegion( body, null );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java
index e7fa47b..59110df 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java
@@ -101,7 +101,7 @@ public class QakkaQueueManager implements LegacyQueueManager {
@Override
- public <T extends Serializable> void sendMessageToLocalRegion(T body) throws IOException {
+ public <T extends Serializable> void sendMessageToLocalRegion(T body, Boolean async) throws IOException {
List<String> regionsList = regions.getRegions( Regions.LOCAL );
logger.trace( "Sending message to queue {} local region {}", scope.getName(), regionsList );
doSendMessage( body, regionsList );
@@ -109,7 +109,7 @@ public class QakkaQueueManager implements LegacyQueueManager {
@Override
- public <T extends Serializable> void sendMessageToAllRegions(T body) throws IOException {
+ public <T extends Serializable> void sendMessageToAllRegions(T body, Boolean async) throws IOException {
List<String> regionsList = regions.getRegions( Regions.ALL );
logger.trace( "Sending message to queue {} all regions {}", scope.getName(), regionsList );
doSendMessage( body, regionsList );
@@ -183,12 +183,16 @@ public class QakkaQueueManager implements LegacyQueueManager {
}
}
+ @Override
+ public void sendMessagesAsync( List bodies ) throws IOException {
+ sendMessages(bodies);
+ }
@Override
public void sendMessages( List bodies ) throws IOException {
for ( Object body : bodies ) {
- sendMessageToLocalRegion( (Serializable)body );
+ sendMessageToLocalRegion( (Serializable)body, null );
}
}
@@ -199,7 +203,7 @@ public class QakkaQueueManager implements LegacyQueueManager {
List<LegacyQueueMessage> successMessages = new ArrayList<>();
for ( LegacyQueueMessage queueMessage : queueMessages ) {
- sendMessageToLocalRegion( (Serializable)queueMessage.getBody() );
+ sendMessageToLocalRegion( (Serializable)queueMessage.getBody() , null);
successMessages.add(queueMessage);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index 5b49bc7..775a64b 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -541,7 +541,49 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
@Override
- public <T extends Serializable> void sendMessageToAllRegions(final T body ) throws IOException {
+ public <T extends Serializable> void sendMessageToAllRegions(final T body, Boolean async) throws IOException {
+ boolean sendAsync = async == null ? fig.isAsyncQueue() : async.booleanValue();
+ if (sendAsync) {
+ sendMessageToAllRegionsAsync(body);
+ } else {
+ sendMessageToAllRegionsSync(body);
+ }
+ }
+
+
+ private <T extends Serializable> void sendMessageToAllRegionsSync(final T body) throws IOException {
+ if ( sns == null ) {
+ logger.error( "SNS client is null, perhaps it failed to initialize successfully" );
+ return;
+ }
+
+ final String stringBody = toString( body );
+
+ String topicArn = getWriteTopicArn();
+
+ if ( logger.isTraceEnabled() ) {
+ logger.trace( "Publishing Message...{} to arn: {}", stringBody, topicArn );
+ }
+
+ try {
+ PublishResult publishResult = sns.publish(topicArn, toString(body));
+ if ( logger.isTraceEnabled() ) {
+ logger.trace( "Successfully published... messageID=[{}], arn=[{}]", publishResult.getMessageId(),
+ topicArn );
+ }
+ } catch (Exception e) {
+ if (logger.isErrorEnabled()) {
+ logger.error("Failed to send this message {} to SNS queue at {}", stringBody, topicArn);
+ }
+ sendMessageToAllRegionsAsync(body);
+ }
+
+
+
+ }
+
+
+ private <T extends Serializable> void sendMessageToAllRegionsAsync(final T body ) throws IOException {
if ( snsAsync == null ) {
logger.error( "SNS client is null, perhaps it failed to initialize successfully" );
return;
@@ -574,17 +616,27 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
} );
}
-
@Override
- public void sendMessages( final List bodies ) throws IOException {
-
+ public void sendMessagesAsync( final List bodies ) throws IOException {
if ( sqsAsync == null ) {
logger.error( "SQS client is null, perhaps it failed to initialize successfully" );
return;
}
for ( Object body : bodies ) {
- sendMessageToLocalRegion( ( Serializable ) body );
+ sendMessageToLocalRegionAsync( ( Serializable ) body );
+ }
+ }
+
+
+ @Override
+ public void sendMessages( final List bodies ) throws IOException {
+ for ( Object body : bodies ) {
+ if (fig.isAsyncQueue()) {
+ sendMessageToLocalRegionAsync((Serializable) body);
+ } else {
+ sendMessageToLocalRegionSync((Serializable) body);
+ }
}
}
@@ -625,16 +677,57 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
return successMessages;
}
-
@Override
- public <T extends Serializable> void sendMessageToLocalRegion(final T body ) throws IOException {
+ public <T extends Serializable> void sendMessageToLocalRegion(final T body, Boolean async) throws IOException {
+ boolean sendAsync = async == null ? fig.isAsyncQueue() : async.booleanValue();
+ if (sendAsync) {
+ sendMessageToLocalRegionAsync(body);
+ } else {
+ sendMessageToLocalRegionSync(body);
+ }
+ }
- if ( sqsAsync == null ) {
+ private <T extends Serializable> void sendMessageToLocalRegionSync(final T body) throws IOException {
+
+ if ( sqs == null ) {
logger.error( "SQS client is null, perhaps it failed to initialize successfully" );
return;
}
final String stringBody = toString( body );
+ if (logger.isDebugEnabled()) {
+ logger.debug(" sendMessageToLocalRegion " + stringBody);
+ }
+
+ String url = getReadQueue().getUrl();
+
+ if ( logger.isTraceEnabled() ) {
+ logger.trace( "Publishing Message...{} to url: {}", stringBody, url );
+ }
+
+ SendMessageRequest messageRequest = new SendMessageRequest(url, stringBody);
+ try {
+ SendMessageResult result = sqs.sendMessage(messageRequest);
+ if (logger.isTraceEnabled()) {
+ logger.trace("Successfully published... messageID=[{}], arn=[{}]", result.getMessageId(),
+ url);
+ }
+ } catch (Exception e) {
+ logger.error("Failed to send this message {}. To this address {}. Error was ", messageRequest.getMessageBody(), url, e);
+ sendMessageToLocalRegionAsync(body);
+ }
+
+
+ }
+
+
+ private <T extends Serializable> void sendMessageToLocalRegionAsync(final T body ) throws IOException {
+
+ if ( sqsAsync == null ) {
+ logger.error( "SQS client is null, perhaps it failed to initialize successfully" );
+ return;
+ }
+ final String stringBody = toString( body );
String url = getReadQueue().getUrl();
if ( logger.isTraceEnabled() ) {
@@ -647,8 +740,7 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
@Override
public void onError( final Exception e ) {
-
- logger.error( "Error sending message... {}", e );
+ logger.error("Failed to send this message {}. To this address {}. Error was ", stringBody, url, e);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java
index ad73dd7..c57ff8b 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java
@@ -62,7 +62,7 @@ public class LegacyQueueManagerTest extends AbstractAkkaTest {
LegacyQueueManager qm = qmf.getQueueManager(scope);
String value = "bodytest";
- qm.sendMessageToLocalRegion(value);
+ qm.sendMessageToLocalRegion(value, null);
Thread.sleep(5000);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index d26cd5f..b7afc2e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -356,7 +356,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
if(logger.isTraceEnabled()) {
logger.trace("Queueing notification message for device: {}", message.get().getDeviceId());
}
- qm.sendMessageToLocalRegion( message.get() );
+ qm.sendMessageToLocalRegion( message.get() , null );
queueMeter.mark();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java
index f7e107d..ae58d7d 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java
@@ -59,6 +59,10 @@ public class ImportQueueManager implements LegacyQueueManager {
}
+ @Override
+ public void sendMessagesAsync( final List bodies ) throws IOException {
+
+ }
@Override
public void sendMessages( final List bodies ) throws IOException {
@@ -73,13 +77,13 @@ public class ImportQueueManager implements LegacyQueueManager {
@Override
- public <T extends Serializable> void sendMessageToLocalRegion(final T body ) throws IOException {
+ public <T extends Serializable> void sendMessageToLocalRegion(final T body, Boolean async) throws IOException {
}
@Override
- public <T extends Serializable> void sendMessageToAllRegions(final T body ) throws IOException {
+ public <T extends Serializable> void sendMessageToAllRegions(final T body, Boolean async) throws IOException {
}
[27/27] usergrid git commit: Initial commit for exposing collection
re-index to non sysadmin users.
Posted by mr...@apache.org.
Initial commit for exposing collection re-index to non sysadmin users.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ec8e5c7c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ec8e5c7c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ec8e5c7c
Branch: refs/heads/expose-reindex
Commit: ec8e5c7ca42b52dc128a653bd7b6c46eebda11fd
Parents: 4a65910
Author: Michael Russo <ru...@google.com>
Authored: Wed Jun 28 11:07:15 2017 -0700
Committer: Michael Russo <ru...@google.com>
Committed: Wed Oct 4 19:02:59 2017 -0700
----------------------------------------------------------------------
.../corepersistence/index/ReIndexService.java | 33 +++++-
.../index/ReIndexServiceImpl.java | 106 +++++++++++++------
.../rest/applications/CollectionResource.java | 27 ++++-
.../usergrid/rest/system/IndexResource.java | 78 ++++++++++++--
4 files changed, 196 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ec8e5c7c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index b9238e5..d37f117 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -20,6 +20,8 @@
package org.apache.usergrid.corepersistence.index;
+import org.apache.usergrid.utils.StringUtils;
+
/**
* An interface for re-indexing all entities in an application
*/
@@ -47,6 +49,13 @@ public interface ReIndexService {
*/
ReIndexStatus getStatus( final String jobId );
+ /**
+ * Get the status of a collection job
+ * @param collectionName The collectionName for the rebuild index
+ * @return
+ */
+ ReIndexStatus getStatusForCollection( final String appIdString, final String collectionName );
+
/**
* The response when requesting a re-index operation
@@ -56,14 +65,27 @@ public interface ReIndexService {
final Status status;
final long numberProcessed;
final long lastUpdated;
+ final String collectionName;
public ReIndexStatus( final String jobId, final Status status, final long numberProcessed,
- final long lastUpdated ) {
- this.jobId = jobId;
+ final long lastUpdated, final String collectionName ) {
+
+ if(StringUtils.isNotEmpty(jobId)){
+ this.jobId = jobId;
+ }else {
+ this.jobId = "";
+ }
+
this.status = status;
this.numberProcessed = numberProcessed;
this.lastUpdated = lastUpdated;
+
+ if(StringUtils.isNotEmpty(collectionName)){
+ this.collectionName = collectionName;
+ }else {
+ this.collectionName = "";
+ }
}
@@ -74,6 +96,13 @@ public interface ReIndexService {
return jobId;
}
+ /**
+ * Get the jobId used to resume this operation
+ */
+ public String getCollectionName() {
+ return collectionName;
+ }
+
/**
* Get the last updated time, as a long
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ec8e5c7c/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 05602fc..d4fb249 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -75,6 +75,7 @@ public class ReIndexServiceImpl implements ReIndexService {
private static final String MAP_COUNT_KEY = "count";
private static final String MAP_STATUS_KEY = "status";
private static final String MAP_UPDATED_KEY = "lastUpdated";
+ private static final String MAP_SEPARATOR = "|||";
private final AllApplicationsObservable allApplicationsObservable;
@@ -140,7 +141,9 @@ public class ReIndexServiceImpl implements ReIndexService {
// create an observable that loads a batch to be indexed
- if(reIndexRequestBuilder.getCollectionName().isPresent()) {
+ final boolean isForCollection = reIndexRequestBuilder.getCollectionName().isPresent();
+
+ if(isForCollection) {
String collectionName = InflectionUtils.pluralize(
CpNamingUtils.getNameFromEdgeType(reIndexRequestBuilder.getCollectionName().get() ));
@@ -175,12 +178,36 @@ public class ReIndexServiceImpl implements ReIndexService {
if( edgeScopes.size() > 0 ) {
writeCursorState(jobId, edgeScopes.get(edgeScopes.size() - 1));
}
- writeStateMeta( jobId, Status.INPROGRESS, count.get(), System.currentTimeMillis() ); })
- .doOnCompleted(() -> writeStateMeta( jobId, Status.COMPLETE, count.get(), System.currentTimeMillis() ))
+ if( isForCollection ){
+ writeStateMetaForCollection(
+ appId.get().getApplication().getUuid().toString(),
+ reIndexRequestBuilder.getCollectionName().get(),
+ Status.INPROGRESS, count.get(),
+ System.currentTimeMillis() );
+ }else{
+ writeStateMeta( jobId, Status.INPROGRESS, count.get(), System.currentTimeMillis() );
+ }
+ })
+ .doOnCompleted(() ->{
+ if( isForCollection ){
+ writeStateMetaForCollection(
+ appId.get().getApplication().getUuid().toString(),
+ reIndexRequestBuilder.getCollectionName().get(),
+ Status.COMPLETE, count.get(),
+ System.currentTimeMillis() );
+ }else {
+ writeStateMeta(jobId, Status.COMPLETE, count.get(), System.currentTimeMillis());
+ }
+ })
.subscribeOn( Schedulers.io() ).subscribe();
+ if(isForCollection){
+ return new ReIndexStatus( "", Status.STARTED, 0, 0, reIndexRequestBuilder.getCollectionName().get() );
+
+ }
+
- return new ReIndexStatus( jobId, Status.STARTED, 0, 0 );
+ return new ReIndexStatus( jobId, Status.STARTED, 0, 0, "" );
}
@@ -196,38 +223,15 @@ public class ReIndexServiceImpl implements ReIndexService {
return getIndexResponse( jobId );
}
-
- /**
- * Simple collector that counts state, then flushed every time a buffer is provided. Writes final state when complete
- */
- private class FlushingCollector {
-
- private final String jobId;
- private long count;
-
-
- private FlushingCollector( final String jobId ) {
- this.jobId = jobId;
- }
-
-
- public void flushBuffer( final List<EdgeScope> buffer ) {
- count += buffer.size();
-
- //write our cursor state
- if ( buffer.size() > 0 ) {
- writeCursorState( jobId, buffer.get( buffer.size() - 1 ) );
- }
-
- writeStateMeta( jobId, Status.INPROGRESS, count, System.currentTimeMillis() );
- }
-
- public void complete(){
- writeStateMeta( jobId, Status.COMPLETE, count, System.currentTimeMillis() );
- }
+ @Override
+ public ReIndexStatus getStatusForCollection( final String appIdString, final String collectionName ) {
+ Preconditions.checkNotNull( collectionName, "appIdString must not be null" );
+ Preconditions.checkNotNull( collectionName, "collectionName must not be null" );
+ return getIndexResponseForCollection( appIdString, collectionName );
}
+
/**
* Get the resume edge scope
*
@@ -346,7 +350,7 @@ public class ReIndexServiceImpl implements ReIndexService {
final String stringStatus = mapManager.getString( jobId+MAP_STATUS_KEY );
if(stringStatus == null){
- return new ReIndexStatus( jobId, Status.UNKNOWN, 0, 0 );
+ return new ReIndexStatus( jobId, Status.UNKNOWN, 0, 0, "" );
}
final Status status = Status.valueOf( stringStatus );
@@ -354,7 +358,39 @@ public class ReIndexServiceImpl implements ReIndexService {
final long processedCount = mapManager.getLong( jobId + MAP_COUNT_KEY );
final long lastUpdated = mapManager.getLong( jobId + MAP_UPDATED_KEY );
- return new ReIndexStatus( jobId, status, processedCount, lastUpdated );
+ return new ReIndexStatus( jobId, status, processedCount, lastUpdated, "" );
+ }
+
+
+ private void writeStateMetaForCollection(final String appIdString, final String collectionName,
+ final Status status, final long processedCount, final long lastUpdated ) {
+
+ if(logger.isDebugEnabled()) {
+ logger.debug( "Flushing state for collection {}, status {}, processedCount {}, lastUpdated {}",
+ collectionName, status, processedCount, lastUpdated);
+ }
+
+ mapManager.putString( appIdString + MAP_SEPARATOR + collectionName + MAP_STATUS_KEY, status.name() );
+ mapManager.putLong( appIdString + MAP_SEPARATOR + collectionName + MAP_COUNT_KEY, processedCount );
+ mapManager.putLong( appIdString + MAP_SEPARATOR + collectionName + MAP_UPDATED_KEY, lastUpdated );
+ }
+
+
+ private ReIndexStatus getIndexResponseForCollection( final String appIdString, final String collectionName ) {
+
+ final String stringStatus =
+ mapManager.getString( appIdString + MAP_SEPARATOR + collectionName + MAP_STATUS_KEY );
+
+ if(stringStatus == null){
+ return new ReIndexStatus( "", Status.UNKNOWN, 0, 0, collectionName );
+ }
+
+ final Status status = Status.valueOf( stringStatus );
+
+ final long processedCount = mapManager.getLong( appIdString + MAP_SEPARATOR + collectionName + MAP_COUNT_KEY );
+ final long lastUpdated = mapManager.getLong( appIdString + MAP_SEPARATOR + collectionName + MAP_UPDATED_KEY );
+
+ return new ReIndexStatus( "", status, processedCount, lastUpdated, collectionName );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ec8e5c7c/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
index 9c09806..6294cec 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
@@ -48,7 +48,10 @@ import org.apache.usergrid.services.ServicePayload;
import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+<<<<<<< HEAD
import java.util.HashMap;
+=======
+>>>>>>> Initial commit for exposing collection re-index to non sysadmin users.
import java.util.Map;
@@ -254,20 +257,21 @@ public class CollectionResource extends ServiceResource {
// TODO: this can't be controlled and until it can be controlled we shouldn' allow muggles to do this.
// So system access only.
// TODO: use scheduler here to get around people sending a reindex call 30 times.
+
@POST
@Path("{itemName}/_reindex")
@Produces({ MediaType.APPLICATION_JSON,"application/javascript"})
- @RequireSystemAccess
+ @RequireApplicationAccess
@JSONP
public ApiResponse executePostForReindexing(
- @Context UriInfo ui, String body,
+ @Context UriInfo ui, final Map<String, Object> payload,
@PathParam("itemName") PathSegment itemName,
@QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {
addItemToServiceContext( ui, itemName );
IndexResource indexResource = new IndexResource(injector);
- return indexResource.rebuildIndexesPost(
+ return indexResource.rebuildIndexCollectionPost(payload,
services.getApplicationId().toString(),itemName.getPath(),false,callback );
}
@@ -322,6 +326,23 @@ public class CollectionResource extends ServiceResource {
response.setSuccess();
return response;
+=======
+ @GET
+ @Path("{itemName}/_reindex")
+ @Produces({ MediaType.APPLICATION_JSON,"application/javascript"})
+ @RequireApplicationAccess
+ @JSONP
+ public ApiResponse executeGetForReindexStatus(
+ @Context UriInfo ui, final Map<String, Object> payload,
+ @PathParam("itemName") PathSegment itemName,
+ @QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {
+
+ addItemToServiceContext( ui, itemName );
+
+ IndexResource indexResource = new IndexResource(injector);
+ return indexResource.rebuildIndexCollectionGet(services.getApplicationId().toString(), itemName.getPath(),
+ callback );
+>>>>>>> Initial commit for exposing collection re-index to non sysadmin users.
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ec8e5c7c/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
index be60177..ec86750 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/system/IndexResource.java
@@ -28,13 +28,16 @@ import com.google.inject.Injector;
import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl;
import org.apache.usergrid.corepersistence.index.ReIndexService;
+import org.apache.usergrid.exception.ConflictException;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.index.utils.ConversionUtils;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.RootResource;
+import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
import org.apache.usergrid.rest.security.annotations.RequireSystemAccess;
+import org.apache.usergrid.utils.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
@@ -182,26 +185,78 @@ public class IndexResource extends AbstractContextResource {
return executeResumeAndCreateResponse( payload, request, callback );
}
+ @RequireOrganizationAccess
+ @GET
+ @Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
+ @JSONP
+ @Produces({ MediaType.APPLICATION_JSON, "application/javascript" })
+ public ApiResponse rebuildIndexCollectionGet( @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
- @RequireSystemAccess
+
+ throws Exception {
+ if (logger.isTraceEnabled()) {
+ logger.trace("Getting re-index status for app: {}, collection: {}", applicationIdStr, collectionName);
+ }
+
+
+ ReIndexService.ReIndexStatus status = getReIndexService().getStatusForCollection(applicationIdStr, collectionName);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "get rebuild index status" );
+ response.setProperty( "collection", status.getCollectionName() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return response;
+ }
+
+ @RequireOrganizationAccess
@POST
@Path( "rebuild/" + RootResource.APPLICATION_ID_PATH + "/{collectionName}" )
@JSONP
@Produces({MediaType.APPLICATION_JSON, "application/javascript"})
- public ApiResponse rebuildIndexesPost( @PathParam( "applicationId" ) final String applicationIdStr,
- @PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
- @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
+ public ApiResponse rebuildIndexCollectionPost(final Map<String, Object> payload,
+ @PathParam( "applicationId" ) final String applicationIdStr,
+ @PathParam( "collectionName" ) final String collectionName,
+ @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
+ @QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
+ ReIndexService.ReIndexStatus existingStatus =
+ getReIndexService().getStatusForCollection(applicationIdStr, collectionName);
- logger.info( "Rebuilding collection {} in application {}", collectionName, applicationIdStr );
+ if(existingStatus.getStatus().equals(ReIndexService.Status.INPROGRESS)){
+ throw new ConflictException("Re-index for collection currently in progress");
+ }
+
+ logger.info( "Re-indexing collection {} in application {}", collectionName, applicationIdStr );
final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+
final ReIndexRequestBuilder request =
createRequest().withApplicationId( appId ).withCollection( collectionName );
+ Map<String,Object> newPayload = payload;
+ if(newPayload == null || !payload.containsKey( UPDATED_FIELD )){
+ newPayload = new HashMap<>(1);
+ newPayload.put(UPDATED_FIELD,0);
+ }
+
+ Preconditions.checkArgument(newPayload.get(UPDATED_FIELD) instanceof Number,
+ "Property \"updated\" in the payload must be a number in unix timestamp millis format" );
+
+ //add our updated timestamp to the request
+ if ( newPayload.containsKey( UPDATED_FIELD ) ) {
+ final long timestamp = ConversionUtils.getLong(newPayload.get(UPDATED_FIELD));
+ request.withStartTimestamp( timestamp );
+ }
+
return executeAndCreateResponse( request, callback );
}
@@ -214,7 +269,6 @@ public class IndexResource extends AbstractContextResource {
public ApiResponse rebuildIndexesPut( final Map<String, Object> payload,
@PathParam( "applicationId" ) final String applicationIdStr,
@PathParam( "collectionName" ) final String collectionName,
- @QueryParam( "reverse" ) @DefaultValue( "false" ) final Boolean reverse,
@QueryParam( "callback" ) @DefaultValue( "callback" ) String callback )
throws Exception {
@@ -350,7 +404,15 @@ public class IndexResource extends AbstractContextResource {
final ApiResponse response = createApiResponse();
response.setAction( "rebuild indexes" );
- response.setProperty( "jobId", status.getJobId() );
+
+ if(StringUtils.isNotEmpty(status.getJobId())){
+ response.setProperty( "jobId", status.getJobId() );
+ }
+
+ if(StringUtils.isNotEmpty(status.getCollectionName())){
+ response.setProperty( "collection", status.getCollectionName() );
+ }
+
response.setProperty( "status", status.getStatus() );
response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
response.setProperty( "numberQueued", status.getNumberProcessed() );
[14/27] usergrid git commit: Converts de-indexing of edges and
entities to identify documents in the index to delete based on data in
Cassandra vs. resource intensive queries to the index. Fixes issue where
nodes were not actually getting deleted from gr
Posted by mr...@apache.org.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c201f1f6/stack/websocket/pom.xml
----------------------------------------------------------------------
diff --git a/stack/websocket/pom.xml b/stack/websocket/pom.xml
index af5ed56..72a5c9d 100644
--- a/stack/websocket/pom.xml
+++ b/stack/websocket/pom.xml
@@ -70,7 +70,7 @@
<storage-config>${basedir}/src/test/conf</storage-config>
</systemPropertyVariables>
<forkMode>always</forkMode>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -Dfile.encoding=UTF-8 -Dsun.jnu.encoding=UTF-8 -javaagent:${settings.localRepository}/org/jacoco/org.jacoco.agent/${jacoco.version}/org.jacoco.agent-${jacoco.version}-runtime.jar=destfile=${project.build.directory}/jacoco.exec -javaagent:${settings.localRepository}/com/github/jbellis/jamm/0.3.0/jamm-0.3.0.jar ${ug.argline}</argLine>
</configuration>
</plugin>
</plugins>
[05/27] usergrid git commit: fix html encode in jsp
Posted by mr...@apache.org.
fix html encode in jsp
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/be2f69c3
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/be2f69c3
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/be2f69c3
Branch: refs/heads/expose-reindex
Commit: be2f69c37cab7214382d99e36e27347fc9f3b09b
Parents: 068263e
Author: Mike Dunker <md...@google.com>
Authored: Fri Jul 28 14:11:31 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 09:47:00 2017 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/rest/TestResource/error.jsp | 4 ++--
.../org/apache/usergrid/rest/TestResource/test.jsp | 5 +++--
.../ApplicationResource/authorize_form.jsp | 15 ++++++++-------
.../rest/applications/ApplicationResource/error.jsp | 4 ++--
.../applications/users/UserResource/activate.jsp | 4 ++--
.../rest/applications/users/UserResource/confirm.jsp | 4 ++--
.../rest/applications/users/UserResource/error.jsp | 4 ++--
.../users/UserResource/resetpw_email_form.jsp | 11 ++++++-----
.../users/UserResource/resetpw_email_success.jsp | 4 ++--
.../users/UserResource/resetpw_set_form.jsp | 9 +++++----
.../users/UserResource/resetpw_set_success.jsp | 4 ++--
.../rest/applications/users/UsersResource/error.jsp | 4 ++--
.../users/UsersResource/resetpw_email_form.jsp | 9 +++++----
.../users/UsersResource/resetpw_email_success.jsp | 4 ++--
.../management/ManagementResource/authorize_form.jsp | 15 ++++++++-------
.../rest/management/ManagementResource/error.jsp | 4 ++--
.../organizations/OrganizationResource/activate.jsp | 4 ++--
.../organizations/OrganizationResource/confirm.jsp | 4 ++--
.../organizations/OrganizationResource/error.jsp | 4 ++--
.../rest/management/users/UserResource/activate.jsp | 4 ++--
.../rest/management/users/UserResource/confirm.jsp | 4 ++--
.../rest/management/users/UserResource/error.jsp | 4 ++--
.../users/UserResource/resetpw_email_form.jsp | 9 +++++----
.../users/UserResource/resetpw_email_success.jsp | 4 ++--
.../users/UserResource/resetpw_set_form.jsp | 8 ++++----
.../users/UserResource/resetpw_set_success.jsp | 4 ++--
.../rest/management/users/UsersResource/error.jsp | 4 ++--
.../users/UsersResource/resetpw_email_form.jsp | 6 +++---
.../users/UsersResource/resetpw_email_success.jsp | 4 ++--
29 files changed, 87 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
index 83a6ad1..68c12f2 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/TestResource/test.jsp
@@ -1,5 +1,6 @@
<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -24,6 +25,6 @@ limitations under the License.
<link rel="stylesheet" type="text/css" href="/css/styles.css" />
</head>
<body>
-<h1>${it.foo}</h1>
+<h1>${fn:escapeXml(it.foo)}</h1>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
index 6b1b8b2..0079bcf 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/authorize_form.jsp
@@ -2,6 +2,7 @@
pageEncoding="ISO-8859-1"%>
<%@ page import="org.apache.usergrid.rest.AbstractContextResource"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -28,13 +29,13 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
- <input type="hidden" name="response_type" value="${it.responseType}">
- <input type="hidden" name="client_id" value="${it.clientId}">
- <input type="hidden" name="redirect_uri" value="${it.redirectUri}">
- <input type="hidden" name="scope" value="${it.scope}">
- <input type="hidden" name="state" value="${it.state}">
+ <input type="hidden" name="response_type" value="${fn:escapeXml(it.responseType)}">
+ <input type="hidden" name="client_id" value="${fn:escapeXml(it.clientId)}">
+ <input type="hidden" name="redirect_uri" value="${fn:escapeXml(it.redirectUri)}">
+ <input type="hidden" name="scope" value="${fn:escapeXml(it.scope)}">
+ <input type="hidden" name="state" value="${fn:escapeXml(it.state)}">
<fieldset>
<p>
<label for="username">Username</label>
@@ -56,4 +57,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/ApplicationResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp
index dfcf3b7..20e69b8 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/activate.jsp
@@ -26,7 +26,7 @@ limitations under the License.
</head>
<body>
- <p>Your account with email address <c:out value="${it.user.email}"/> has been successfully activated.</p>
+ <p>Your account with email address <c:out value="${it.user.email}" escapeXml="true"/> has been successfully activated.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp
index 02e9ee3..d7f3acc 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/confirm.jsp
@@ -26,8 +26,8 @@ limitations under the License.
</head>
<body>
- <p>Your account with email address <c:out value="${it.user.email}"/> has been successfully confirmed.
+ <p>Your account with email address <c:out value="${it.user.email}" escapeXml="true"/> has been successfully confirmed.
You will received an email soon to let you know when you account has been activated</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
index 0f53bfc..59026bf 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_form.jsp
@@ -1,6 +1,7 @@
<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -26,17 +27,17 @@ limitations under the License.
</head>
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}">
- <div class="dialog-form-message">${it.errorMsg}</div>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
<p>
Enter the captcha to have your password reset instructions sent to
- <c:out value="${it.user.email}" />
+ <c:out value="${it.user.email}" escapeXml="true" />
</p>
<p id="human-proof"></p>
- ${it.reCaptchaHtml}
+ ${fn:escapeXml(it.reCaptchaHtml)}
<p class="buttons">
<input type="submit" value="submit" />
</p>
@@ -44,4 +45,4 @@ limitations under the License.
</form>
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp
index 23f8508..41c5176 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_email_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>Email with instructions for password reset sent to <c:out value="${it.user.email}"/></p>
+ <p>Email with instructions for password reset sent to <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
index a83d80d..19b6528 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_form.jsp
@@ -4,6 +4,7 @@
<%@ page import="net.tanesha.recaptcha.ReCaptchaFactory"%>
<%@ page import="org.apache.usergrid.rest.AbstractContextResource"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -30,12 +31,12 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
- <input type="hidden" name="token" value="${it.token}">
+ <input type="hidden" name="token" value="${fn:escapeXml(it.token)}">
<fieldset>
<p>
- <label for="password1">Please enter your new password for <c:out value="${it.user.email}"/>.</label>
+ <label for="password1">Please enter your new password for <c:out value="${it.user.email}" escapeXml="true"/>.</label>
</p>
<p>
<input class="text_field" id="password1" name="password1" type="password" />
@@ -54,4 +55,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp
index 9de90ba..3915084 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UserResource/resetpw_set_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>New password set for <c:out value="${it.user.email}"/></p>
+ <p>New password set for <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
index 3211a3a..f86240f 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_form.jsp
@@ -1,6 +1,7 @@
<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -27,8 +28,8 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}">
- <div class="dialog-form-message">${it.errorMsg}</div>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
@@ -41,7 +42,7 @@ limitations under the License.
<input class="text_field" id="email" name="email" type="text" />
</p>
<p id="human-proof"></p>
- ${it.reCaptchaHtml}
+ ${fn:escapeXml(it.reCaptchaHtml)}
<p class="buttons">
<button type="submit">Submit</button>
</p>
@@ -50,4 +51,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp
index 23f8508..41c5176 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/applications/users/UsersResource/resetpw_email_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>Email with instructions for password reset sent to <c:out value="${it.user.email}"/></p>
+ <p>Email with instructions for password reset sent to <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
index 6b1b8b2..0079bcf 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/authorize_form.jsp
@@ -2,6 +2,7 @@
pageEncoding="ISO-8859-1"%>
<%@ page import="org.apache.usergrid.rest.AbstractContextResource"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -28,13 +29,13 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
- <input type="hidden" name="response_type" value="${it.responseType}">
- <input type="hidden" name="client_id" value="${it.clientId}">
- <input type="hidden" name="redirect_uri" value="${it.redirectUri}">
- <input type="hidden" name="scope" value="${it.scope}">
- <input type="hidden" name="state" value="${it.state}">
+ <input type="hidden" name="response_type" value="${fn:escapeXml(it.responseType)}">
+ <input type="hidden" name="client_id" value="${fn:escapeXml(it.clientId)}">
+ <input type="hidden" name="redirect_uri" value="${fn:escapeXml(it.redirectUri)}">
+ <input type="hidden" name="scope" value="${fn:escapeXml(it.scope)}">
+ <input type="hidden" name="state" value="${fn:escapeXml(it.state)}">
<fieldset>
<p>
<label for="username">Username</label>
@@ -56,4 +57,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/ManagementResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp
index 85114cd..f5fa14d 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/activate.jsp
@@ -26,7 +26,7 @@ limitations under the License.
</head>
<body>
- <p>Your organization <c:out value="${it.organization.name}"/> has been successfully activated.</p>
+ <p>Your organization <c:out value="${it.organization.name}" escapeXml="true"/> has been successfully activated.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp
index f4307b7..5fb41c7 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/confirm.jsp
@@ -26,8 +26,8 @@ limitations under the License.
</head>
<body>
- <p>Your organization <c:out value="${it.organization.name}"/> has been successfully confirmed.
+ <p>Your organization <c:out value="${it.organization.name}" escapeXml="true"/> has been successfully confirmed.
You will received an email soon to let you know when you organization has been activated</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/organizations/OrganizationResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp
index dfcf3b7..20e69b8 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/activate.jsp
@@ -26,7 +26,7 @@ limitations under the License.
</head>
<body>
- <p>Your account with email address <c:out value="${it.user.email}"/> has been successfully activated.</p>
+ <p>Your account with email address <c:out value="${it.user.email}" escapeXml="true"/> has been successfully activated.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp
index 02e9ee3..d7f3acc 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/confirm.jsp
@@ -26,8 +26,8 @@ limitations under the License.
</head>
<body>
- <p>Your account with email address <c:out value="${it.user.email}"/> has been successfully confirmed.
+ <p>Your account with email address <c:out value="${it.user.email}" escapeXml="true"/> has been successfully confirmed.
You will received an email soon to let you know when you account has been activated</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
index 3e56cd1..c9f8309 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_form.jsp
@@ -1,6 +1,7 @@
<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -26,17 +27,17 @@ limitations under the License.
</head>
<body>
<div class="dialog-area password-reset-form">
- <c:if test="${!empty it.errorMsg}">
- <div class="dialog-form-message">${it.errorMsg}</div>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}">
+ <div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div>
</c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
<p>
Enter the captcha to have your password reset instructions sent to
- <c:out value="${it.user.email}" />
+ <c:out value="${it.user.email}" escapeXml="true" />
</p>
<p id="human-proof"></p>
- ${it.reCaptchaHtml}
+ ${fn:escapeXml(it.reCaptchaHtml)}
<p class="buttons">
<input type="submit" value="submit" />
</p>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp
index 23f8508..41c5176 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_email_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>Email with instructions for password reset sent to <c:out value="${it.user.email}"/></p>
+ <p>Email with instructions for password reset sent to <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
index a83d80d..60384c4 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
@@ -30,12 +30,12 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
- <input type="hidden" name="token" value="${it.token}">
+ <input type="hidden" name="token" value="${fn:escapeXml(it.token)}">
<fieldset>
<p>
- <label for="password1">Please enter your new password for <c:out value="${it.user.email}"/>.</label>
+ <label for="password1">Please enter your new password for <c:out value="${it.user.email}" escapeXml="true"/>.</label>
</p>
<p>
<input class="text_field" id="password1" name="password1" type="password" />
@@ -54,4 +54,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp
index 9de90ba..3915084 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>New password set for <c:out value="${it.user.email}"/></p>
+ <p>New password set for <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp
index be184b1..d02ad40 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/error.jsp
@@ -27,7 +27,7 @@ limitations under the License.
</head>
<body>
- <p>An error occurred <c:out value="${it}"/>.</p>
+ <p>An error occurred <c:out value="${it}" escapeXml="true"/>.</p>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
index 8643016..8b15cd4 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
@@ -27,7 +27,7 @@ limitations under the License.
<body>
<div class="dialog-area">
- <c:if test="${!empty it.errorMsg}"><div class="dialog-form-message">${it.errorMsg}</div></c:if>
+ <c:if test="${!empty fn:escnapeXml(it.errorMsg)}"><div class="dialog-form-message">${fn:escapeXml(it.errorMsg)}</div></c:if>
<form class="dialog-form" action="" method="post">
<fieldset>
<p>
@@ -38,7 +38,7 @@ limitations under the License.
<input class="text_field" id="email" name="email" type="text" />
</p>
<p id="human-proof"></p>
- ${it.reCaptchaHtml}
+ ${fn:escapeXml(it.reCaptchaHtml)}
<p class="buttons">
<button type="submit">Submit</button>
</p>
@@ -47,4 +47,4 @@ limitations under the License.
</div>
</body>
-</html>
\ No newline at end of file
+</html>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be2f69c3/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp
index 23f8508..41c5176 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_success.jsp
@@ -29,7 +29,7 @@ limitations under the License.
</head>
<body>
- <p>Email with instructions for password reset sent to <c:out value="${it.user.email}"/></p>
+ <p>Email with instructions for password reset sent to <c:out value="${it.user.email}" escapeXml="true"/></p>
</body>
-</html>
\ No newline at end of file
+</html>
[19/27] usergrid git commit: Give entity creation time to finish
during collection delete test.
Posted by mr...@apache.org.
Give entity creation time to finish during collection delete test.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3d3ba490
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3d3ba490
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3d3ba490
Branch: refs/heads/expose-reindex
Commit: 3d3ba49065018a74a51709ed23bb777826b21f78
Parents: 8e9eff8
Author: Mike Dunker <md...@google.com>
Authored: Mon Aug 28 19:29:06 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Aug 28 19:29:06 2017 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/CollectionDeleteTest.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3d3ba490/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
index ddf2c68..d062ef4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
@@ -111,6 +111,9 @@ public class CollectionDeleteTest extends AbstractCoreIT {
}
logger.info("Created {} entities", ENTITIES_TO_DELETE);
+
+ app.waitForQueueDrainAndRefreshIndex(10000);
+
long timeFirstPutDone = System.currentTimeMillis();
logger.info("timeFirstPutDone={}", timeFirstPutDone);
[07/27] usergrid git commit: add password complexity check before
submitting during reset password flow
Posted by mr...@apache.org.
add password complexity check before submitting during reset password flow
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/70de6fde
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/70de6fde
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/70de6fde
Branch: refs/heads/expose-reindex
Commit: 70de6fde4438bd4ba750806e5b4a23cd9aa07d9b
Parents: 459163b
Author: Mike Dunker <md...@google.com>
Authored: Thu Aug 17 11:02:36 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Aug 17 11:02:36 2017 -0700
----------------------------------------------------------------------
.../usergrid/rest/applications/users/UserResource.java | 9 +++++++++
.../usergrid/rest/management/users/UserResource.java | 9 +++++++++
.../org/apache/usergrid/management/ManagementService.java | 10 +++++-----
.../management/cassandra/ManagementServiceImpl.java | 10 ++++++++++
4 files changed, 33 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
index 5435f7e..3e4542d 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/users/UserResource.java
@@ -17,6 +17,7 @@
package org.apache.usergrid.rest.applications.users;
+import java.util.Collection;
import java.util.Map;
import java.util.UUID;
@@ -465,6 +466,14 @@ public class UserResource extends ServiceResource {
if ( ( password1 != null ) || ( password2 != null ) ) {
if ( management.checkPasswordResetTokenForAppUser( getApplicationId(), getUserUuid(), token ) ) {
if ( ( password1 != null ) && password1.equals( password2 ) ) {
+ // validate password
+ Collection<String> violations = management.passwordPolicyCheck(password1, false);
+ if (violations.size() > 0) {
+ // password not valid
+ errorMsg = management.getPasswordDescription(false);
+ return handleViewable("resetpw_set_form", this, getOrganizationName());
+ }
+
management.setAppUserPassword( getApplicationId(), getUser().getUuid(), password1 );
management.revokeAccessTokenForAppUser( token );
return handleViewable( "resetpw_set_success", this, getOrganizationName() );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
index b747aa4..1f80bc1 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/users/UserResource.java
@@ -43,6 +43,7 @@ import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.UriInfo;
+import java.util.Collection;
import java.util.Map;
import java.util.UUID;
@@ -297,6 +298,14 @@ public class UserResource extends AbstractContextResource {
if ( ( password1 != null ) || ( password2 != null ) ) {
if ( management.checkPasswordResetTokenForAdminUser( user.getUuid(), tokenInfo ) ) {
if ( ( password1 != null ) && password1.equals( password2 ) ) {
+ // validate password
+ Collection<String> violations = management.passwordPolicyCheck(password1, true);
+ if (violations.size() > 0) {
+ // password not valid
+ errorMsg = management.getPasswordDescription(true);
+ return handleViewable( "resetpw_set_form", this, organizationId );
+ }
+
management.setAdminUserPassword( user.getUuid(), password1 );
management.revokeAccessTokenForAdminUser( user.getUuid(), token );
loginEndpoint = properties.getProperty("usergrid.viewable.loginEndpoint");
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
index 2b88b07..8b840d6 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ManagementService.java
@@ -17,11 +17,7 @@
package org.apache.usergrid.management;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.UUID;
+import java.util.*;
import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.Entity;
@@ -372,6 +368,10 @@ public interface ManagementService {
Observable<Id> deleteAllEntities(final UUID applicationId,final int limit);
+ Collection<String> passwordPolicyCheck(String password, boolean isAdminUser);
+
+ String getPasswordDescription(boolean isAdminUser);
+
// DO NOT REMOVE BELOW METHODS, THEY ARE HERE TO ALLOW EXTERNAL CLASSES TO OVERRIDE AND HOOK INTO POST PROCESSING
void createOrganizationPostProcessing( final OrganizationInfo orgInfo,
http://git-wip-us.apache.org/repos/asf/usergrid/blob/70de6fde/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 2ba9bde..89375fd 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -3412,6 +3412,16 @@ public class ManagementServiceImpl implements ManagementService {
return service.deleteAllEntities(CpNamingUtils.getApplicationScope(applicationId),limit);
}
+ @Override
+ public Collection<String> passwordPolicyCheck(String password, boolean isAdminUser) {
+ return passwordPolicy.policyCheck(password, isAdminUser);
+ }
+
+ @Override
+ public String getPasswordDescription(boolean isAdminUser) {
+ return passwordPolicy.getDescription(isAdminUser);
+ }
+
private String getProperty(String key) {
String obj = properties.getProperty(key);
if(StringUtils.isEmpty(obj))
[17/27] usergrid git commit: Add ability to walk through a collection
and delete all the entities,
optionally up to a certain timestamp. Modeled after reindex services.
Posted by mr...@apache.org.
Add ability to walk through a collection and delete all the entities, optionally up to a certain timestamp. Modeled after reindex services.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/11823f29
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/11823f29
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/11823f29
Branch: refs/heads/expose-reindex
Commit: 11823f294dfae762754ef1c4da8a5ee573107968
Parents: 3f7afcd
Author: Mike Dunker <md...@google.com>
Authored: Mon Aug 28 14:46:17 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Aug 28 14:46:17 2017 -0700
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 1 +
.../asyncevents/AsyncEventService.java | 3 +-
.../asyncevents/AsyncEventServiceImpl.java | 67 +++--
.../asyncevents/EventBuilder.java | 13 +-
.../asyncevents/EventBuilderImpl.java | 45 ++-
.../asyncevents/model/EntityDeleteEvent.java | 27 +-
.../index/CollectionDeleteAction.java | 43 +++
.../index/CollectionDeleteRequestBuilder.java | 92 ++++++
.../CollectionDeleteRequestBuilderImpl.java | 146 +++++++++
.../index/CollectionDeleteService.java | 108 +++++++
.../index/CollectionDeleteServiceImpl.java | 299 +++++++++++++++++++
.../index/IndexProcessorFig.java | 9 +
.../index/ReIndexServiceImpl.java | 2 +-
.../persistence/CollectionDeleteTest.java | 266 +++++++++++++++++
.../resources/usergrid-custom-test.properties | 2 +
.../rest/applications/CollectionResource.java | 130 +++++++-
16 files changed, 1217 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index ec6b775..a0748e6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -140,6 +140,7 @@ public class CoreModule extends AbstractModule {
bind( ReIndexService.class ).to( ReIndexServiceImpl.class );
+ bind( CollectionDeleteService.class ).to( CollectionDeleteServiceImpl.class );
bind( ExportService.class ).to( ExportServiceImpl.class );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index 9e346cf..04eaf4c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.asyncevents;
+import org.apache.usergrid.corepersistence.index.CollectionDeleteAction;
import org.apache.usergrid.corepersistence.index.ReIndexAction;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
@@ -33,7 +34,7 @@ import java.util.UUID;
/**
* Low level queue service for events in the entity. These events are fire and forget, and will always be asynchronous
*/
-public interface AsyncEventService extends ReIndexAction {
+public interface AsyncEventService extends ReIndexAction, CollectionDeleteAction {
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 428772f..3d06cae 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -75,9 +75,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import static org.apache.commons.lang.StringUtils.indexOf;
-import static org.apache.commons.lang.StringUtils.isNotEmpty;
-
/**
* TODO, this whole class is becoming a nightmare.
@@ -106,7 +103,6 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public static final String QUEUE_NAME = "index"; //keep this short as AWS limits queue name size to 80 chars
public static final String QUEUE_NAME_UTILITY = "utility"; //keep this short as AWS limits queue name size to 80 chars
public static final String QUEUE_NAME_DELETE = "delete";
- public static final String DEAD_LETTER_SUFFIX = "_dead";
private final LegacyQueueManager indexQueue;
@@ -522,8 +518,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
applicationScope);
- logger.trace("Offering InitializeApplicationIndexEvent for {}:{}",
- applicationScope.getApplication().getUuid(), applicationScope.getApplication().getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering InitializeApplicationIndexEvent for {}:{}",
+ applicationScope.getApplication().getUuid(), applicationScope.getApplication().getType());
+ }
offerTopic( new InitializeApplicationIndexEvent( queueFig.getPrimaryRegion(),
new ReplicatedIndexLocationStrategy( indexLocationStrategy ) ), AsyncEventQueueType.REGULAR);
@@ -535,8 +533,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final Entity entity, long updatedAfter) {
- logger.trace("Offering EntityIndexEvent for {}:{}",
- entity.getId().getUuid(), entity.getId().getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering EntityIndexEvent for {}:{}",
+ entity.getId().getUuid(), entity.getId().getType());
+ }
offer(new EntityIndexEvent(queueFig.getPrimaryRegion(),
new EntityIdScope(applicationScope, entity.getId()), updatedAfter));
@@ -577,8 +577,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final Entity entity,
final Edge newEdge) {
- logger.trace("Offering EdgeIndexEvent for edge type {} entity {}:{}",
- newEdge.getType(), entity.getId().getUuid(), entity.getId().getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering EdgeIndexEvent for edge type {} entity {}:{}",
+ newEdge.getType(), entity.getId().getUuid(), entity.getId().getType());
+ }
offer( new EdgeIndexEvent( queueFig.getPrimaryRegion(), applicationScope, entity.getId(), newEdge ));
@@ -612,8 +614,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public void queueDeleteEdge(final ApplicationScope applicationScope,
final Edge edge) {
- logger.trace("Offering EdgeDeleteEvent for type {} to target {}:{}",
- edge.getType(), edge.getTargetNode().getUuid(), edge.getTargetNode().getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering EdgeDeleteEvent for type {} to target {}:{}",
+ edge.getType(), edge.getTargetNode().getUuid(), edge.getTargetNode().getType());
+ }
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ), AsyncEventQueueType.DELETE );
@@ -675,7 +679,9 @@ public class AsyncEventServiceImpl implements AsyncEventService {
//send to the topic so all regions index the batch
- logger.trace("Offering ElasticsearchIndexEvent for message {}", newMessageId );
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering ElasticsearchIndexEvent for message {}", newMessageId);
+ }
offerTopic( elasticsearchIndexEvent, queueType );
}
@@ -749,8 +755,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// queue the de-index of old versions to the topic so cleanup happens in all regions
- logger.trace("Offering DeIndexOldVersionsEvent for app {} {}:{}",
- applicationScope.getApplication().getUuid(), entityId.getUuid(), entityId.getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering DeIndexOldVersionsEvent for app {} {}:{}",
+ applicationScope.getApplication().getUuid(), entityId.getUuid(), entityId.getType());
+ }
offerTopic( new DeIndexOldVersionsEvent( queueFig.getPrimaryRegion(),
new EntityIdScope( applicationScope, entityId), markedVersion), AsyncEventQueueType.DELETE );
@@ -810,7 +818,9 @@ public class AsyncEventServiceImpl implements AsyncEventService {
@Override
public void queueEntityDelete(final ApplicationScope applicationScope, final Id entityId) {
- logger.trace("Offering EntityDeleteEvent for {}:{}", entityId.getUuid(), entityId.getType());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering EntityDeleteEvent for {}:{}", entityId.getUuid(), entityId.getType());
+ }
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
offer( new EntityDeleteEvent(queueFig.getPrimaryRegion(), new EntityIdScope( applicationScope, entityId ) ),
@@ -830,12 +840,15 @@ public class AsyncEventServiceImpl implements AsyncEventService {
final EntityDeleteEvent entityDeleteEvent = ( EntityDeleteEvent ) event;
final ApplicationScope applicationScope = entityDeleteEvent.getEntityIdScope().getApplicationScope();
final Id entityId = entityDeleteEvent.getEntityIdScope().getId();
+ final boolean isCollectionDelete = entityDeleteEvent.isCollectionDelete();
+ final long updatedBefore = entityDeleteEvent.getUpdatedBefore();
if (logger.isDebugEnabled()) {
- logger.debug("Deleting entity id from index in app scope {} with entityId {}", applicationScope, entityId);
+ logger.debug("Deleting entity id from index in app scope {} with entityId {}, isCollectionDelete {}, updatedBefore {}",
+ applicationScope, entityId, isCollectionDelete, updatedBefore);
}
- return eventBuilder.buildEntityDelete( applicationScope, entityId );
+ return eventBuilder.buildEntityDelete( applicationScope, entityId, isCollectionDelete, updatedBefore );
}
@@ -1192,11 +1205,27 @@ public class AsyncEventServiceImpl implements AsyncEventService {
});
- logger.trace("Offering batch of EntityIndexEvent of size {}", batch.size());
+ if (logger.isTraceEnabled()) {
+ logger.trace("Offering batch of EntityIndexEvent of size {}", batch.size());
+ }
offerBatch( batch, queueType );
}
+ public void deleteBatch(final List<EdgeScope> edges, final long updatedBefore, AsyncEventQueueType queueType) {
+
+ final List<EntityDeleteEvent> batch = new ArrayList<>();
+ edges.forEach(e -> {
+
+ //change to id scope to avoid serialization issues
+ batch.add(new EntityDeleteEvent(queueFig.getPrimaryRegion(),
+ new EntityIdScope(e.getApplicationScope(), e.getEdge().getTargetNode()), true, updatedBefore));
+
+ });
+
+ offerBatch(batch, queueType);
+ }
+
public class IndexEventResult{
private final Optional<IndexOperationMessage> indexOperationMessage;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
index ebb9190..4bb6312 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilder.java
@@ -63,7 +63,18 @@ public interface EventBuilder {
* @param entityId
* @return
*/
- IndexOperationMessage buildEntityDelete(ApplicationScope applicationScope, Id entityId );
+ IndexOperationMessage buildEntityDelete(ApplicationScope applicationScope, Id entityId);
+
+ /**
+ * Return a bin with 2 observable streams for entity delete.
+ * @param applicationScope
+ * @param entityId
+ * @param isCollectionDelete
+ * @param updatedBefore
+ * @return
+ */
+ IndexOperationMessage buildEntityDelete(ApplicationScope applicationScope, Id entityId,
+ boolean isCollectionDelete, long updatedBefore);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 5051598..7c72b72 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -24,7 +24,9 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import org.antlr.misc.Graph;
import org.apache.usergrid.corepersistence.index.*;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -128,15 +130,48 @@ public class EventBuilderImpl implements EventBuilder {
//it'll need to be pushed up higher so we can do the marking that isn't async or does it not matter?
@Override
- public IndexOperationMessage buildEntityDelete(final ApplicationScope applicationScope, final Id entityId ) {
+ public IndexOperationMessage buildEntityDelete(final ApplicationScope applicationScope, final Id entityId) {
+ return buildEntityDelete(applicationScope, entityId, false, Long.MAX_VALUE);
+ }
+
+ @Override
+ public IndexOperationMessage buildEntityDelete(final ApplicationScope applicationScope, final Id entityId,
+ final boolean isCollectionDelete, final long updatedBefore) {
if (logger.isDebugEnabled()) {
- logger.debug("Deleting entity id (marked versions) from index in app scope {} with entityId {}",
- applicationScope, entityId);
+ logger.debug("Deleting entity id (marked versions) from index in app scope {} with entityId {}, isCollectionDelete {}, updatedBefore={}",
+ applicationScope, entityId, isCollectionDelete, updatedBefore);
+ }
+
+ final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager(applicationScope);
+ final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
+
+ boolean deleteEntity = ecm.load(entityId).
+ map(entity -> {
+ final Field<Long> modified = entity.getField( Schema.PROPERTY_MODIFIED );
+
+ boolean willDelete = false;
+ if ( modified == null ) {
+ // We don't have a modified field, so we can't check, so delete it
+ willDelete = true;
+ } else if (modified.getValue() <= updatedBefore) {
+ willDelete = true;
+ }
+
+ if (isCollectionDelete && willDelete) {
+ // need to mark for deletion
+ ecm.mark(entityId, null)
+ .mergeWith(gm.markNode(entityId, CpNamingUtils.createGraphOperationTimestamp()))
+ .toBlocking().last();
+ }
+
+ return willDelete;
+ }).toBlocking().firstOrDefault(true);
+
+ if (!deleteEntity) {
+ return new IndexOperationMessage();
}
- final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
- final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
MvccLogEntry mostRecentToDelete =
ecm.getVersionsFromMaxToMin( entityId, UUIDUtils.newTimeUUID() )
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java
index 01d2ba8..1589632 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/model/EntityDeleteEvent.java
@@ -24,6 +24,7 @@ import org.apache.usergrid.persistence.collection.serialization.impl.migration.E
/**
* Event that will signal to finish the actual delete (post-mark delete) for an Entity
+ * It will mark if this is for a collection delete
*/
public final class EntityDeleteEvent extends AsyncEvent {
@@ -31,17 +32,41 @@ public final class EntityDeleteEvent extends AsyncEvent {
@JsonProperty
protected EntityIdScope entityIdScope;
+ @JsonProperty
+ private long updatedBefore;
+
+ @JsonProperty
+ private boolean isCollectionDelete;
+
public EntityDeleteEvent() {
super();
}
public EntityDeleteEvent(String sourceRegion, EntityIdScope entityIdScope) {
super(sourceRegion);
- this.entityIdScope = entityIdScope;
+ this.entityIdScope = entityIdScope;
+ this.updatedBefore = Long.MAX_VALUE;
+ this.isCollectionDelete = false;
+ }
+
+ public EntityDeleteEvent(String sourceRegion, EntityIdScope entityIdScope,
+ boolean isCollectionDelete, long updatedBefore) {
+ super(sourceRegion);
+ this.entityIdScope = entityIdScope;
+ this.updatedBefore = updatedBefore;
+ this.isCollectionDelete = isCollectionDelete;
}
public EntityIdScope getEntityIdScope() {
return entityIdScope;
}
+
+ public long getUpdatedBefore() {
+ return updatedBefore;
+ }
+
+ public boolean isCollectionDelete() {
+ return isCollectionDelete;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteAction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteAction.java
new file mode 100644
index 0000000..7bad06b
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteAction.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import java.util.List;
+
+
+/**
+ * Callback to perform a collection delete operation based on an scope during bulk collection delete operations
+ */
+public interface CollectionDeleteAction {
+
+ /**
+ * Delete a batch list of entities.
+ * @param edges
+ * @param updatedBefore
+ * @param queueType
+ */
+ void deleteBatch(final List<EdgeScope> edges, final long updatedBefore, AsyncEventQueueType queueType);
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilder.java
new file mode 100644
index 0000000..4abdfea
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilder.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import com.google.common.base.Optional;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+
+/**
+ * A builder interface to build our collection delete request
+ */
+public interface CollectionDeleteRequestBuilder {
+
+ /**
+ * Set the application id
+ */
+ CollectionDeleteRequestBuilder withApplicationId(final UUID applicationId);
+
+ /**
+ * Set the collection name.
+ * @param collectionName
+ * @return
+ */
+ CollectionDeleteRequestBuilder withCollection(final String collectionName);
+
+ /**
+ * Set our cursor to resume processing
+ * @param cursor
+ * @return
+ */
+ CollectionDeleteRequestBuilder withCursor(final String cursor);
+
+
+ CollectionDeleteRequestBuilder withDelay(int delayTimer, TimeUnit timeUnit);
+
+ /**
+ * Set the timestamp to delete entities updated <= this timestamp
+ * @param timestamp
+ * @return
+ */
+ CollectionDeleteRequestBuilder withEndTimestamp(final Long timestamp);
+
+
+ Optional<Integer> getDelayTimer();
+
+ Optional<TimeUnit> getTimeUnitOptional();
+
+ /**
+ * Get the application scope
+ * @return
+ */
+ Optional<ApplicationScope> getApplicationScope();
+
+ /**
+ * Get the collection name
+ * @return
+ */
+ Optional<String> getCollectionName();
+
+ /**
+ * Get the cursor
+ * @return
+ */
+ Optional<String> getCursor();
+
+ /**
+ * Get the latest timestamp to delete
+ * @return
+ */
+ Optional<Long> getEndTimestamp();
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilderImpl.java
new file mode 100644
index 0000000..890b770
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteRequestBuilderImpl.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import com.google.common.base.Optional;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+
+/**
+ * collection delete service request builder
+ */
+public class CollectionDeleteRequestBuilderImpl implements CollectionDeleteRequestBuilder {
+
+ private Optional<UUID> withApplicationId = Optional.absent();
+ private Optional<String> withCollectionName = Optional.absent();
+ private Optional<String> cursor = Optional.absent();
+ private Optional<Long> endTimestamp = Optional.absent();
+ private Optional<Integer> delayTimer = Optional.absent();
+ private Optional<TimeUnit> timeUnitOptional = Optional.absent();
+
+
+ /***
+ *
+ * @param applicationId The application id
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withApplicationId( final UUID applicationId ) {
+ this.withApplicationId = Optional.fromNullable( applicationId );
+ return this;
+ }
+
+
+ /**
+ * the collection name
+ * @param collectionName
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withCollection( final String collectionName ) {
+ this.withCollectionName = Optional.fromNullable( CpNamingUtils.getEdgeTypeFromCollectionName( collectionName.toLowerCase() ) );
+ return this;
+ }
+
+
+ /**
+ * The cursor
+ * @param cursor
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withCursor( final String cursor ) {
+ this.cursor = Optional.fromNullable( cursor );
+ return this;
+ }
+
+
+ /**
+ * Determines whether we should tack on a delay for collection delete and for how long if we do. Also
+ * allowed to specify how throttled back it should be.
+ * @param delayTimer
+ * @param timeUnit
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withDelay( final int delayTimer, final TimeUnit timeUnit ){
+ this.delayTimer = Optional.fromNullable( delayTimer );
+ this.timeUnitOptional = Optional.fromNullable( timeUnit );
+
+ return this;
+ }
+
+
+ /**
+ * Set end timestamp in epoch time. Only entities created before this time will be processed for deletion
+ * @param timestamp
+ * @return
+ */
+ @Override
+ public CollectionDeleteRequestBuilder withEndTimestamp( final Long timestamp ) {
+ this.endTimestamp = Optional.fromNullable( timestamp );
+ return this;
+ }
+
+
+ @Override
+ public Optional<Integer> getDelayTimer() {
+ return delayTimer;
+ }
+
+ @Override
+ public Optional<TimeUnit> getTimeUnitOptional() {
+ return timeUnitOptional;
+ }
+
+
+ @Override
+ public Optional<ApplicationScope> getApplicationScope() {
+
+ if ( this.withApplicationId.isPresent() ) {
+ return Optional.of( CpNamingUtils.getApplicationScope( withApplicationId.get() ) );
+ }
+
+ return Optional.absent();
+ }
+
+
+ @Override
+ public Optional<String> getCollectionName() {
+ return withCollectionName;
+ }
+
+
+ @Override
+ public Optional<String> getCursor() {
+ return cursor;
+ }
+
+
+ @Override
+ public Optional<Long> getEndTimestamp() {
+ return endTimestamp;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteService.java
new file mode 100644
index 0000000..c939dd3
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteService.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+/**
+ * An interface for re-indexing all entities in an application
+ */
+public interface CollectionDeleteService {
+
+
+ /**
+ * Perform a collection delete via service
+ *
+ * @param collectionDeleteRequestBuilder The builder to build the request
+ */
+ CollectionDeleteStatus deleteCollection(final CollectionDeleteRequestBuilder collectionDeleteRequestBuilder);
+
+
+ /**
+ * Generate a build for the collection delete
+ */
+ CollectionDeleteRequestBuilder getBuilder();
+
+
+ /**
+ * Get the status of a job
+ * @param jobId The jobId returned during the collection delete
+ * @return
+ */
+ CollectionDeleteStatus getStatus(final String jobId);
+
+
+ /**
+ * The response when requesting a collection delete operation
+ */
+ public class CollectionDeleteStatus {
+ final String jobId;
+ final Status status;
+ final long numberProcessed;
+ final long lastUpdated;
+
+
+ public CollectionDeleteStatus(final String jobId, final Status status, final long numberProcessed,
+ final long lastUpdated ) {
+ this.jobId = jobId;
+ this.status = status;
+ this.numberProcessed = numberProcessed;
+ this.lastUpdated = lastUpdated;
+ }
+
+
+ /**
+ * Get the jobId used to resume this operation
+ */
+ public String getJobId() {
+ return jobId;
+ }
+
+
+ /**
+ * Get the last updated time, as a long
+ * @return
+ */
+ public long getLastUpdated() {
+ return lastUpdated;
+ }
+
+
+ /**
+ * Get the number of records processed
+ * @return
+ */
+ public long getNumberProcessed() {
+ return numberProcessed;
+ }
+
+
+ /**
+ * Get the status
+ * @return
+ */
+ public Status getStatus() {
+ return status;
+ }
+ }
+
+ enum Status{
+ STARTED, INPROGRESS, COMPLETE, UNKNOWN;
+ }
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteServiceImpl.java
new file mode 100644
index 0000000..7b3e324
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionDeleteServiceImpl.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.index;
+
+
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializerUtil;
+import org.apache.usergrid.corepersistence.pipeline.read.CursorSeek;
+import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.AllEntityIdsObservable;
+import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.StringUtils;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.map.MapManager;
+import org.apache.usergrid.persistence.map.MapManagerFactory;
+import org.apache.usergrid.persistence.map.MapScope;
+import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.apache.usergrid.utils.JsonUtils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+import rx.schedulers.Schedulers;
+
+import static com.google.common.base.Optional.fromNullable;
+
+
+@Singleton
+public class CollectionDeleteServiceImpl implements CollectionDeleteService {
+
+ private static final Logger logger = LoggerFactory.getLogger( CollectionDeleteServiceImpl.class );
+
+ private static final MapScope RESUME_MAP_SCOPE =
+ new MapScopeImpl( CpNamingUtils.getManagementApplicationId(), "collectiondeleteresume" );
+
+ //Keep cursors to resume collection delete for 10 days.
+ private static final int CURSOR_TTL = 60 * 60 * 24 * 10;
+
+ private static final String MAP_CURSOR_KEY = "cursor";
+ private static final String MAP_COUNT_KEY = "count";
+ private static final String MAP_STATUS_KEY = "status";
+ private static final String MAP_UPDATED_KEY = "lastUpdated";
+
+
+ private final AllApplicationsObservable allApplicationsObservable;
+ private final IndexLocationStrategyFactory indexLocationStrategyFactory;
+ private final AllEntityIdsObservable allEntityIdsObservable;
+ private final IndexProcessorFig indexProcessorFig;
+ private final MapManager mapManager;
+ private final MapManagerFactory mapManagerFactory;
+ private final AsyncEventService indexService;
+ private final EntityIndexFactory entityIndexFactory;
+ private final CollectionSettingsFactory collectionSettingsFactory;
+
+
+ @Inject
+ public CollectionDeleteServiceImpl(final EntityIndexFactory entityIndexFactory,
+ final IndexLocationStrategyFactory indexLocationStrategyFactory,
+ final AllEntityIdsObservable allEntityIdsObservable,
+ final MapManagerFactory mapManagerFactory,
+ final AllApplicationsObservable allApplicationsObservable,
+ final IndexProcessorFig indexProcessorFig,
+ final CollectionSettingsFactory collectionSettingsFactory,
+ final AsyncEventService indexService ) {
+ this.entityIndexFactory = entityIndexFactory;
+ this.indexLocationStrategyFactory = indexLocationStrategyFactory;
+ this.allEntityIdsObservable = allEntityIdsObservable;
+ this.allApplicationsObservable = allApplicationsObservable;
+ this.indexProcessorFig = indexProcessorFig;
+ this.indexService = indexService;
+ this.collectionSettingsFactory = collectionSettingsFactory;
+ this.mapManagerFactory = mapManagerFactory;
+ this.mapManager = mapManagerFactory.createMapManager( RESUME_MAP_SCOPE );
+ }
+
+
+ //TODO: optional delay, param.
+ @Override
+ public CollectionDeleteStatus deleteCollection( final CollectionDeleteRequestBuilder collectionDeleteRequestBuilder) {
+
+ final AtomicInteger count = new AtomicInteger();
+
+ final Optional<EdgeScope> cursor = parseCursor( collectionDeleteRequestBuilder.getCursor() );
+
+ final CursorSeek<Edge> cursorSeek = getResumeEdge( cursor );
+
+ final Optional<Integer> delayTimer = collectionDeleteRequestBuilder.getDelayTimer();
+
+ final Optional<TimeUnit> timeUnitOptional = collectionDeleteRequestBuilder.getTimeUnitOptional();
+
+ Optional<ApplicationScope> appId = collectionDeleteRequestBuilder.getApplicationScope();
+
+ Preconditions.checkArgument(collectionDeleteRequestBuilder.getCollectionName().isPresent(),
+ "You must specify a collection name");
+ String collectionName = collectionDeleteRequestBuilder.getCollectionName().get();
+
+ Preconditions.checkArgument( !(cursor.isPresent() && appId.isPresent()),
+ "You cannot specify an app id and a cursor. When resuming with cursor you must omit the appid." );
+ Preconditions.checkArgument( cursor.isPresent() || appId.isPresent(),
+ "Either application ID or cursor is required.");
+
+ ApplicationScope applicationScope;
+ if (appId.isPresent()) {
+ applicationScope = appId.get();
+ } else { // cursor is present
+ applicationScope = cursor.get().getApplicationScope();
+ }
+
+
+ final String jobId = StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+
+ // default to current time
+ final long endTimestamp = collectionDeleteRequestBuilder.getEndTimestamp().or( System.currentTimeMillis() );
+
+ String pluralizedCollectionName = InflectionUtils.pluralize(CpNamingUtils.getNameFromEdgeType(collectionName));
+
+ CollectionSettings collectionSettings =
+ collectionSettingsFactory.getInstance(new CollectionSettingsScopeImpl(applicationScope.getApplication(), pluralizedCollectionName));
+
+ Optional<Map<String, Object>> existingSettings =
+ collectionSettings.getCollectionSettings( pluralizedCollectionName );
+
+ if ( existingSettings.isPresent() ) {
+
+ Map jsonMapData = existingSettings.get();
+
+ jsonMapData.put( "lastCollectionClear", Instant.now().toEpochMilli() );
+
+ collectionSettings.putCollectionSettings(
+ pluralizedCollectionName, JsonUtils.mapToJsonString(jsonMapData ) );
+ }
+
+ allEntityIdsObservable.getEdgesToEntities( Observable.just(applicationScope),
+ fromNullable(collectionName), cursorSeek.getSeekValue() )
+ .buffer( indexProcessorFig.getCollectionDeleteBufferSize())
+ .doOnNext( edgeScopes -> {
+ logger.info("Sending batch of {} to be deleted.", edgeScopes.size());
+ indexService.deleteBatch(edgeScopes, endTimestamp, AsyncEventQueueType.DELETE);
+ count.addAndGet(edgeScopes.size() );
+ if( edgeScopes.size() > 0 ) {
+ writeCursorState(jobId, edgeScopes.get(edgeScopes.size() - 1));
+ }
+ writeStateMeta( jobId, Status.INPROGRESS, count.get(), System.currentTimeMillis() ); })
+ .doOnCompleted(() -> writeStateMeta( jobId, Status.COMPLETE, count.get(), System.currentTimeMillis() ))
+ .subscribeOn( Schedulers.io() ).subscribe();
+
+
+ return new CollectionDeleteStatus( jobId, Status.STARTED, 0, 0 );
+ }
+
+
+ @Override
+ public CollectionDeleteRequestBuilder getBuilder() {
+ return new CollectionDeleteRequestBuilderImpl();
+ }
+
+
+ @Override
+ public CollectionDeleteStatus getStatus( final String jobId ) {
+ Preconditions.checkNotNull( jobId, "jobId must not be null" );
+ return getCollectionDeleteResponse( jobId );
+ }
+
+
+ /**
+ * Get the resume edge scope
+ *
+ * @param edgeScope The optional edge scope from the cursor
+ */
+ private CursorSeek<Edge> getResumeEdge( final Optional<EdgeScope> edgeScope ) {
+
+
+ if ( edgeScope.isPresent() ) {
+ return new CursorSeek<>( Optional.of( edgeScope.get().getEdge() ) );
+ }
+
+ return new CursorSeek<>( Optional.absent() );
+ }
+
+
+ /**
+ * Swap our cursor for an optional edgescope
+ */
+ private Optional<EdgeScope> parseCursor( final Optional<String> cursor ) {
+
+ if ( !cursor.isPresent() ) {
+ return Optional.absent();
+ }
+
+ //get our cursor
+ final String persistedCursor = mapManager.getString( cursor.get() );
+
+ if ( persistedCursor == null ) {
+ return Optional.absent();
+ }
+
+ final JsonNode node = CursorSerializerUtil.fromString( persistedCursor );
+
+ final EdgeScope edgeScope = EdgeScopeSerializer.INSTANCE.fromJsonNode( node, CursorSerializerUtil.getMapper() );
+
+ return Optional.of( edgeScope );
+ }
+
+
+ /**
+ * Write the cursor state to the map in cassandra
+ */
+ private void writeCursorState( final String jobId, final EdgeScope edge ) {
+
+ final JsonNode node = EdgeScopeSerializer.INSTANCE.toNode( CursorSerializerUtil.getMapper(), edge );
+
+ final String serializedState = CursorSerializerUtil.asString( node );
+
+ mapManager.putString( jobId + MAP_CURSOR_KEY, serializedState, CURSOR_TTL);
+ }
+
+
+ /**
+ * Write our state meta data into cassandra so everyone can see it
+ * @param jobId
+ * @param status
+ * @param processedCount
+ * @param lastUpdated
+ */
+ private void writeStateMeta( final String jobId, final Status status, final long processedCount,
+ final long lastUpdated ) {
+
+ if(logger.isDebugEnabled()) {
+ logger.debug( "Flushing state for jobId {}, status {}, processedCount {}, lastUpdated {}",
+ jobId, status, processedCount, lastUpdated);
+ }
+
+ mapManager.putString( jobId + MAP_STATUS_KEY, status.name() );
+ mapManager.putLong( jobId + MAP_COUNT_KEY, processedCount );
+ mapManager.putLong( jobId + MAP_UPDATED_KEY, lastUpdated );
+ }
+
+
+ /**
+ * Get the index response from the jobId
+ * @param jobId
+ * @return
+ */
+ private CollectionDeleteStatus getCollectionDeleteResponse( final String jobId ) {
+
+ final String stringStatus = mapManager.getString( jobId+MAP_STATUS_KEY );
+
+ if(stringStatus == null){
+ return new CollectionDeleteStatus( jobId, Status.UNKNOWN, 0, 0 );
+ }
+
+ final Status status = Status.valueOf( stringStatus );
+
+ final long processedCount = mapManager.getLong( jobId + MAP_COUNT_KEY );
+ final long lastUpdated = mapManager.getLong( jobId + MAP_UPDATED_KEY );
+
+ return new CollectionDeleteStatus( jobId, status, processedCount, lastUpdated );
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index eb63056..948e106 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -56,6 +56,8 @@ public interface IndexProcessorFig extends GuicyFig {
String REINDEX_BUFFER_SIZE = "elasticsearch.reindex.buffer_size";
+ String COLLECTION_DELETE_BUFFER_SIZE = "elasticsearch.collection_delete.buffer_size";
+
String REINDEX_CONCURRENCY_FACTOR = "elasticsearch.reindex.concurrency.factor";
@@ -157,6 +159,13 @@ public interface IndexProcessorFig extends GuicyFig {
int getReindexConcurrencyFactor();
/**
+ * Number of parallel buffers during collection delete
+ */
+ @Default("500")
+ @Key(COLLECTION_DELETE_BUFFER_SIZE)
+ int getCollectionDeleteBufferSize();
+
+ /**
* Flag to resolve the LOCAL queue implementation service synchronously.
*/
@Default("false")
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index c7371b3..05602fc 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -352,7 +352,7 @@ public class ReIndexServiceImpl implements ReIndexService {
final Status status = Status.valueOf( stringStatus );
final long processedCount = mapManager.getLong( jobId + MAP_COUNT_KEY );
- final long lastUpdated = mapManager.getLong( jobId + MAP_COUNT_KEY );
+ final long lastUpdated = mapManager.getLong( jobId + MAP_UPDATED_KEY );
return new ReIndexStatus( jobId, status, processedCount, lastUpdated );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
new file mode 100644
index 0000000..ddf2c68
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.persistence;
+
+
+import com.codahale.metrics.MetricRegistry;
+import com.google.inject.Injector;
+import net.jcip.annotations.NotThreadSafe;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.cassandra.SpringResource;
+import org.apache.usergrid.corepersistence.index.*;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+import static org.junit.Assert.*;
+
+
+@NotThreadSafe
+public class CollectionDeleteTest extends AbstractCoreIT {
+ private static final Logger logger = LoggerFactory.getLogger( CollectionDeleteTest.class );
+
+ private static final MetricRegistry registry = new MetricRegistry();
+
+
+ private static final int ENTITIES_TO_DELETE = 1000;
+ private static final int ENTITIES_TO_ADD_AFTER_TIME = 3;
+
+
+ @Before
+ public void startReporting() {
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("Starting metrics reporting");
+ }
+ }
+
+
+ @After
+ public void printReport() {
+ logger.debug( "Printing metrics report" );
+ }
+
+
+ @Test( timeout = 240000 )
+ public void clearOneCollection() throws Exception {
+
+ logger.info( "Started clearOneCollection()" );
+
+ String rand = RandomStringUtils.randomAlphanumeric( 5 );
+ final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
+
+ final EntityManager em = setup.getEmf().getEntityManager( appId );
+
+ final CollectionDeleteService collectionDeleteService = setup.getInjector().getInstance( CollectionDeleteService.class );
+
+ // ----------------- create a bunch of entities
+
+ Map<String, Object> entityMap = new HashMap<String, Object>() {{
+ put( "key1", 1000 );
+ put( "key2", 2000 );
+ put( "key3", "Some value" );
+ }};
+
+ String collectionName = "items";
+ String itemType = "item";
+
+
+ List<EntityRef> entityRefs = new ArrayList<EntityRef>();
+ for ( int i = 0; i < ENTITIES_TO_DELETE; i++ ) {
+
+ final Entity entity;
+
+ try {
+ entityMap.put( "key", i );
+ entity = em.create(itemType, entityMap);
+ }
+ catch ( Exception ex ) {
+ throw new RuntimeException( "Error creating entity", ex );
+ }
+
+ entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
+ if ( i % 10 == 0 ) {
+ logger.info( "Created {} entities", i );
+ }
+ }
+
+ logger.info("Created {} entities", ENTITIES_TO_DELETE);
+ long timeFirstPutDone = System.currentTimeMillis();
+ logger.info("timeFirstPutDone={}", timeFirstPutDone);
+
+ for (int i = 0; i < ENTITIES_TO_ADD_AFTER_TIME; i++) {
+
+ final Entity entity;
+
+ try {
+ entityMap.put( "key", ENTITIES_TO_DELETE + i );
+ entity = em.create(itemType, entityMap);
+ }
+ catch ( Exception ex ) {
+ throw new RuntimeException( "Error creating entity", ex );
+ }
+
+ entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
+ if ( i % 10 == 0 ) {
+ logger.info( "Created {} entities after delete time", i );
+ }
+
+ }
+ logger.info("Created {} entities after delete time", ENTITIES_TO_ADD_AFTER_TIME);
+
+
+ app.waitForQueueDrainAndRefreshIndex(5000);
+
+ final CollectionDeleteRequestBuilder builder =
+ collectionDeleteService.getBuilder()
+ .withApplicationId( em.getApplicationId() )
+ .withCollection(collectionName)
+ .withEndTimestamp(timeFirstPutDone);
+
+ CollectionDeleteService.CollectionDeleteStatus status = collectionDeleteService.deleteCollection(builder);
+
+ assertNotNull( status.getJobId(), "JobId is present" );
+
+ logger.info( "Delete collection" );
+
+
+ waitForDelete( status, collectionDeleteService );
+
+ app.waitForQueueDrainAndRefreshIndex(15000);
+
+ // ----------------- test that we can read the entries after the timestamp
+
+ readData( em, collectionName,ENTITIES_TO_ADD_AFTER_TIME);
+ }
+
+ /**
+ * Wait for the delete to occur
+ */
+ private void waitForDelete( final CollectionDeleteService.CollectionDeleteStatus status, final CollectionDeleteService collectionDeleteService )
+ throws InterruptedException, IllegalArgumentException {
+ if (status != null) {
+ logger.info("waitForDelete: jobID={}", status.getJobId());
+ } else {
+ logger.info("waitForDelete: error, status = null");
+ throw new IllegalArgumentException("collectionDeleteStatus = null");
+ }
+ while ( true ) {
+
+ try {
+ final CollectionDeleteService.CollectionDeleteStatus updatedStatus =
+ collectionDeleteService.getStatus( status.getJobId() );
+
+ if (updatedStatus == null) {
+ logger.info("waitForDelete: updated status is null");
+ } else {
+ logger.info("waitForDelete: status={} numberProcessed={}",
+ updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed());
+
+ if ( updatedStatus.getStatus() == CollectionDeleteService.Status.COMPLETE ) {
+ break;
+ }
+ }
+ }
+ catch ( IllegalArgumentException iae ) {
+ //swallow. Thrown if our job can't be found. I.E hasn't updated yet
+ }
+
+
+ Thread.sleep( 1000 );
+ }
+ }
+
+
+ private int readData(EntityManager em, String collectionName, int expectedEntities)
+ throws Exception {
+
+ app.waitForQueueDrainAndRefreshIndex();
+
+ Results results = em.getCollection(em.getApplicationRef(), collectionName, null, expectedEntities,
+ Query.Level.ALL_PROPERTIES, false);
+
+ int count = 0;
+ while ( true ) {
+
+ if (results.getEntities().size() == 0) {
+ break;
+ }
+
+ UUID lastEntityUUID = null;
+ for ( Entity e : results.getEntities() ) {
+
+ assertEquals(2000, e.getProperty("key2"));
+
+ if (count % 100 == 0) {
+ logger.info("read {} entities", count);
+ }
+ lastEntityUUID = e.getUuid();
+ count++;
+ }
+
+ results = em.getCollection(em.getApplicationRef(), collectionName, lastEntityUUID, expectedEntities,
+ Query.Level.ALL_PROPERTIES, false);
+
+ }
+ logger.info("read {} total entities", count);
+
+ assertEquals( "Did not get expected entities", expectedEntities, count );
+ return count;
+ }
+
+ private int countEntities( EntityManager em, String collectionName, int expectedEntities)
+ throws Exception {
+
+ app.waitForQueueDrainAndRefreshIndex();
+
+ Query q = Query.fromQL( "select * where key1=1000" ).withLimit( 1000 );
+ Results results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, expectedEntities );
+
+ int count = 0;
+ while ( true ) {
+
+ count += results.size();
+
+
+ if ( results.hasCursor() ) {
+ logger.info( "Counted {} : query again with cursor", count );
+ q.setCursor( results.getCursor() );
+ results = em.searchCollection( em.getApplicationRef(), collectionName, q );
+ }
+ else {
+ break;
+ }
+ }
+
+ assertEquals( "Did not get expected entities", expectedEntities, count );
+ return count;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/query-validator/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/query-validator/src/test/resources/usergrid-custom-test.properties b/stack/query-validator/src/test/resources/usergrid-custom-test.properties
index bc1ba56..c8e3eee 100644
--- a/stack/query-validator/src/test/resources/usergrid-custom-test.properties
+++ b/stack/query-validator/src/test/resources/usergrid-custom-test.properties
@@ -30,3 +30,5 @@ usergrid.sysadmin.login.allowed=true
# This property is required to be set and cannot be defaulted anywhere
usergrid.cluster_name=usergrid
+
+elasticsearch.queue_impl=LOCAL
http://git-wip-us.apache.org/repos/asf/usergrid/blob/11823f29/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
index b8c1caa..c9174c1 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
@@ -18,19 +18,19 @@
package org.apache.usergrid.rest.applications;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.PathSegment;
import javax.ws.rs.core.UriInfo;
+import com.google.common.base.Preconditions;
+import org.apache.usergrid.corepersistence.index.CollectionDeleteRequestBuilder;
+import org.apache.usergrid.corepersistence.index.CollectionDeleteRequestBuilderImpl;
+import org.apache.usergrid.corepersistence.index.CollectionDeleteService;
+import org.apache.usergrid.persistence.index.utils.ConversionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
@@ -48,6 +48,9 @@ import org.apache.usergrid.services.ServicePayload;
import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
+import java.util.HashMap;
+import java.util.Map;
+
/**
* A collection resource that stands before the Service Resource. If it cannot find
@@ -61,6 +64,9 @@ import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
})
public class CollectionResource extends ServiceResource {
+ private static final Logger logger = LoggerFactory.getLogger( CollectionResource.class );
+ private static final String UPDATED_BEFORE_FIELD = "updatedBefore";
+
public CollectionResource() {
}
@@ -190,6 +196,61 @@ public class CollectionResource extends ServiceResource {
}
+ @PUT
+ @Path("{itemName}/_clear")
+ @Produces({MediaType.APPLICATION_JSON, "application/javascript"})
+ @RequireApplicationAccess
+ @JSONP
+ public ApiResponse clearCollectionPut(
+ final Map<String, Object> payload,
+ @PathParam("itemName") final String collectionName,
+ @QueryParam("callback") @DefaultValue("callback") String callback
+ ) throws Exception {
+
+ logger.info("Clearing collection {} for application {}", collectionName, getApplicationId().toString());
+
+ final CollectionDeleteRequestBuilder request = createRequest()
+ .withApplicationId(getApplicationId())
+ .withCollection(collectionName);
+
+ return executeResumeAndCreateResponse(payload, request, callback);
+
+ }
+
+
+ @GET
+ @Path( "{itemName}/_clear/{jobId}")
+ @Produces({MediaType.APPLICATION_JSON,"application/javascript"})
+ @RequireApplicationAccess
+ @JSONP
+ public ApiResponse clearCollectionJobGet(
+ @Context UriInfo ui,
+ @PathParam("itemName") PathSegment itemName,
+ @PathParam("jobId") String jobId,
+ @QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {
+
+ if(logger.isTraceEnabled()){
+ logger.trace( "CollectionResource.clearCollectionJobGet" );
+ }
+
+ Preconditions
+ .checkNotNull(jobId, "path param jobId must not be null" );
+
+ CollectionDeleteService.CollectionDeleteStatus status = getCollectionDeleteService().getStatus(jobId);
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear collection" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberCheckedForDeletion", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return response;
+ }
+
+
// TODO: this can't be controlled and until it can be controlled we shouldn' allow muggles to do this.
// So system access only.
// TODO: use scheduler here to get around people sending a reindex call 30 times.
@@ -210,4 +271,57 @@ public class CollectionResource extends ServiceResource {
services.getApplicationId().toString(),itemName.getPath(),false,callback );
}
+
+ private CollectionDeleteService getCollectionDeleteService() {
+ return injector.getInstance( CollectionDeleteService.class );
+ }
+
+
+ private CollectionDeleteRequestBuilder createRequest() {
+ return new CollectionDeleteRequestBuilderImpl();
+ }
+
+
+ private ApiResponse executeResumeAndCreateResponse( final Map<String, Object> payload,
+ final CollectionDeleteRequestBuilder request,
+ final String callback ) {
+
+ Map<String,Object> newPayload = payload;
+ if(newPayload == null || !payload.containsKey( UPDATED_BEFORE_FIELD )){
+ newPayload = new HashMap<>(1);
+ newPayload.put(UPDATED_BEFORE_FIELD,Long.MAX_VALUE);
+ }
+
+ Preconditions.checkArgument(newPayload.get(UPDATED_BEFORE_FIELD) instanceof Number,
+ "The field \"updatedBefore\" in the payload must be a timestamp" );
+
+ //add our updated timestamp to the request
+ if ( newPayload.containsKey( UPDATED_BEFORE_FIELD ) ) {
+ final long timestamp = ConversionUtils.getLong(newPayload.get(UPDATED_BEFORE_FIELD));
+ request.withEndTimestamp( timestamp );
+ }
+
+ return executeAndCreateResponse( request, callback );
+ }
+
+ /**
+ * Execute the request and return the response.
+ */
+ private ApiResponse executeAndCreateResponse(final CollectionDeleteRequestBuilder request, final String callback ) {
+
+
+ final CollectionDeleteService.CollectionDeleteStatus status = getCollectionDeleteService().deleteCollection( request );
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "clear collection" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return response;
+ }
+
}
[23/27] usergrid git commit: add missing fn reference in jsps
Posted by mr...@apache.org.
add missing fn reference in jsps
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7f40d685
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7f40d685
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7f40d685
Branch: refs/heads/expose-reindex
Commit: 7f40d685401397b44e2026ac6bd3af446c02ddbd
Parents: 9e16e2f
Author: Mike Dunker <md...@google.com>
Authored: Thu Sep 21 14:29:05 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Thu Sep 21 14:29:05 2017 -0700
----------------------------------------------------------------------
.../rest/management/users/UserResource/resetpw_set_form.jsp | 1 +
.../rest/management/users/UsersResource/resetpw_email_form.jsp | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7f40d685/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
index 6334466..c0203ce 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UserResource/resetpw_set_form.jsp
@@ -4,6 +4,7 @@
<%@ page import="net.tanesha.recaptcha.ReCaptchaFactory"%>
<%@ page import="org.apache.usergrid.rest.AbstractContextResource"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7f40d685/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
index f8cf496..f494dcf 100644
--- a/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
+++ b/stack/rest/src/main/webapp/WEB-INF/jsp/org/apache/usergrid/rest/management/users/UsersResource/resetpw_email_form.jsp
@@ -1,6 +1,7 @@
<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
<%@ taglib uri="http://java.sun.com/jsp/jstl/core" prefix="c"%>
+<%@ taglib uri="http://java.sun.com/jsp/jstl/functions" prefix="fn"%>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
[13/27] usergrid git commit: move deletes to new delete queue -- read
repair will fix attempts to access deleted entities and connections,
so indexing and collection deletes can proceed more slowly than other types of
changes
Posted by mr...@apache.org.
move deletes to new delete queue -- read repair will fix attempts to access deleted entities and connections, so indexing and collection deletes can proceed more slowly than other types of changes
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/570e1ab4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/570e1ab4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/570e1ab4
Branch: refs/heads/expose-reindex
Commit: 570e1ab4e2d4d356756fedee6b7ed5a1bd2d8e93
Parents: 39ec4f2
Author: Mike Dunker <md...@google.com>
Authored: Fri Aug 18 15:12:56 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Fri Aug 18 15:12:56 2017 -0700
----------------------------------------------------------------------
.../asyncevents/AsyncEventQueueType.java | 35 ++
.../asyncevents/AsyncEventService.java | 4 +-
.../asyncevents/AsyncEventServiceImpl.java | 463 +++++++++----------
.../index/IndexProcessorFig.java | 18 +
.../corepersistence/index/ReIndexAction.java | 5 +-
.../index/ReIndexServiceImpl.java | 3 +-
.../read/traverse/AbstractReadGraphFilter.java | 11 +-
.../AbstractReadReverseGraphFilter.java | 11 +-
8 files changed, 292 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
new file mode 100644
index 0000000..4b91e17
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventQueueType.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.asyncevents;
+
+public enum AsyncEventQueueType {
+ REGULAR ("regular"), UTILITY("utility"), DELETE("delete");
+
+ private String displayName;
+ AsyncEventQueueType(String displayName) {
+ this.displayName = displayName;
+ }
+
+ @Override
+ public String toString() {
+ return displayName;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index cab4e3e..9e346cf 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -84,9 +84,9 @@ public interface AsyncEventService extends ReIndexAction {
/**
*
* @param indexOperationMessage
- * @param forUtilityQueue
+ * @param queueType
*/
- void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage, boolean forUtilityQueue);
+ void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage, AsyncEventQueueType queueType);
/**
* @param applicationScope
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 9501ad3..0e55e9b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -59,6 +59,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.persistence.queue.*;
import org.apache.usergrid.persistence.queue.impl.LegacyQueueScopeImpl;
+import org.apache.usergrid.persistence.queue.impl.SNSQueueManagerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
@@ -104,13 +105,16 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public int MAX_TAKE = 10;
public static final String QUEUE_NAME = "index"; //keep this short as AWS limits queue name size to 80 chars
public static final String QUEUE_NAME_UTILITY = "utility"; //keep this short as AWS limits queue name size to 80 chars
+ public static final String QUEUE_NAME_DELETE = "delete";
public static final String DEAD_LETTER_SUFFIX = "_dead";
private final LegacyQueueManager indexQueue;
private final LegacyQueueManager utilityQueue;
+ private final LegacyQueueManager deleteQueue;
private final LegacyQueueManager indexQueueDead;
private final LegacyQueueManager utilityQueueDead;
+ private final LegacyQueueManager deleteQueueDead;
private final IndexProcessorFig indexProcessorFig;
private final LegacyQueueFig queueFig;
private final IndexProducer indexProducer;
@@ -132,8 +136,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
private final Counter indexErrorCounter;
private final AtomicLong counter = new AtomicLong();
private final AtomicLong counterUtility = new AtomicLong();
+ private final AtomicLong counterDelete = new AtomicLong();
private final AtomicLong counterIndexDead = new AtomicLong();
private final AtomicLong counterUtilityDead = new AtomicLong();
+ private final AtomicLong counterDeleteDead = new AtomicLong();
private final AtomicLong inFlight = new AtomicLong();
private final Histogram messageCycle;
private final MapManager esMapPersistence;
@@ -174,16 +180,24 @@ public class AsyncEventServiceImpl implements AsyncEventService {
LegacyQueueScope utilityQueueScope =
new LegacyQueueScopeImpl(QUEUE_NAME_UTILITY, LegacyQueueScope.RegionImplementation.ALL);
+ LegacyQueueScope deleteQueueScope =
+ new LegacyQueueScopeImpl(QUEUE_NAME_DELETE, LegacyQueueScope.RegionImplementation.ALL);
+
LegacyQueueScope indexQueueDeadScope =
new LegacyQueueScopeImpl(QUEUE_NAME, LegacyQueueScope.RegionImplementation.ALL, true);
LegacyQueueScope utilityQueueDeadScope =
new LegacyQueueScopeImpl(QUEUE_NAME_UTILITY, LegacyQueueScope.RegionImplementation.ALL, true);
+ LegacyQueueScope deleteQueueDeadScope =
+ new LegacyQueueScopeImpl(QUEUE_NAME_DELETE, LegacyQueueScope.RegionImplementation.ALL, true);
+
this.indexQueue = queueManagerFactory.getQueueManager(indexQueueScope);
this.utilityQueue = queueManagerFactory.getQueueManager(utilityQueueScope);
+ this.deleteQueue = queueManagerFactory.getQueueManager(deleteQueueScope);
this.indexQueueDead = queueManagerFactory.getQueueManager(indexQueueDeadScope);
this.utilityQueueDead = queueManagerFactory.getQueueManager(utilityQueueDeadScope);
+ this.deleteQueueDead = queueManagerFactory.getQueueManager(deleteQueueDeadScope);
this.indexProcessorFig = indexProcessorFig;
this.queueFig = queueFig;
@@ -206,34 +220,90 @@ public class AsyncEventServiceImpl implements AsyncEventService {
start();
}
+ private String getQueueName(AsyncEventQueueType queueType) {
+ switch (queueType) {
+ case REGULAR:
+ return QUEUE_NAME;
+
+ case UTILITY:
+ return QUEUE_NAME_UTILITY;
+
+ case DELETE:
+ return QUEUE_NAME_DELETE;
+
+ default:
+ throw new IllegalArgumentException("Invalid queue type: " + queueType.toString());
+ }
+ }
+
+ private LegacyQueueManager getQueue(AsyncEventQueueType queueType) {
+ return getQueue(queueType, false);
+ }
+
+ private LegacyQueueManager getQueue(AsyncEventQueueType queueType, boolean isDeadQueue) {
+ switch (queueType) {
+ case REGULAR:
+ return isDeadQueue ? indexQueueDead : indexQueue;
+
+ case UTILITY:
+ return isDeadQueue ? utilityQueueDead : utilityQueue;
+
+ case DELETE:
+ return isDeadQueue ? deleteQueueDead : deleteQueue;
+
+ default:
+ throw new IllegalArgumentException("Invalid queue type: " + queueType.toString());
+ }
+ }
+
+ private AtomicLong getCounter(AsyncEventQueueType queueType, boolean isDeadQueue) {
+ switch (queueType) {
+ case REGULAR:
+ return isDeadQueue ? counterIndexDead : counter;
+
+ case UTILITY:
+ return isDeadQueue ? counterUtilityDead : counterUtility;
+
+ case DELETE:
+ return isDeadQueue ? counterDeleteDead : counterDelete;
+
+ default:
+ throw new IllegalArgumentException("Invalid queue type: " + queueType.toString());
+ }
+ }
+
+
+
+
/**
* Offer the EntityIdScope to SQS
*/
private void offer(final Serializable operation) {
+ offer(operation, AsyncEventQueueType.REGULAR);
+ }
+
+ private void offer(final Serializable operation, AsyncEventQueueType queueType) {
final Timer.Context timer = this.writeTimer.time();
try {
//signal to SQS
- this.indexQueue.sendMessageToLocalRegion( operation );
+ getQueue(queueType).sendMessageToLocalRegion(operation);
} catch (IOException e) {
throw new RuntimeException("Unable to queue message", e);
} finally {
timer.stop();
}
+
}
- private void offerTopic(final Serializable operation, boolean forUtilityQueue) {
+ private void offerTopic(final Serializable operation, AsyncEventQueueType queueType) {
final Timer.Context timer = this.writeTimer.time();
try {
//signal to SQS
- if (forUtilityQueue) {
- this.utilityQueue.sendMessageToAllRegions(operation);
- } else {
- this.indexQueue.sendMessageToAllRegions(operation);
- }
+ getQueue(queueType).sendMessageToAllRegions(operation);
}
catch ( IOException e ) {
throw new RuntimeException( "Unable to queue message", e );
@@ -244,15 +314,11 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
- private void offerBatch(final List operations, boolean forUtilityQueue){
+ private void offerBatch(final List operations, AsyncEventQueueType queueType){
final Timer.Context timer = this.writeTimer.time();
try {
//signal to SQS
- if( forUtilityQueue ){
- this.utilityQueue.sendMessages(operations);
- }else{
- this.indexQueue.sendMessages(operations);
- }
+ getQueue(queueType).sendMessages(operations);
} catch (IOException e) {
throw new RuntimeException("Unable to queue message", e);
} finally {
@@ -260,25 +326,16 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
}
- private void offerBatchToUtilityQueue(final List operations){
- try {
- //signal to SQS
- this.utilityQueue.sendMessages(operations);
- } catch (IOException e) {
- throw new RuntimeException("Unable to queue message", e);
- }
- }
-
/**
* Take message
*/
- private List<LegacyQueueMessage> take() {
+ private List<LegacyQueueMessage> take(AsyncEventQueueType queueType, boolean isDeadQueue) {
final Timer.Context timer = this.readTimer.time();
try {
- return indexQueue.getMessages(MAX_TAKE, AsyncEvent.class);
+ return getQueue(queueType, isDeadQueue).getMessages(MAX_TAKE, AsyncEvent.class);
}
finally {
//stop our timer
@@ -286,52 +343,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
}
- /**
- * Take message from SQS utility queue
- */
- private List<LegacyQueueMessage> takeFromUtilityQueue() {
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
- return utilityQueue.getMessages(MAX_TAKE, AsyncEvent.class);
- }
- finally {
- //stop our timer
- timer.stop();
- }
- }
-
- /**
- * Take message from index dead letter queue
- */
- private List<LegacyQueueMessage> takeFromIndexDeadQueue() {
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
- return indexQueueDead.getMessages(MAX_TAKE, AsyncEvent.class);
- }
- finally {
- //stop our timer
- timer.stop();
- }
- }
-
- /**
- * Take message from SQS utility dead letter queue
- */
- private List<LegacyQueueMessage> takeFromUtilityDeadQueue() {
-
- final Timer.Context timer = this.readTimer.time();
-
- try {
- return utilityQueueDead.getMessages(MAX_TAKE, AsyncEvent.class);
- }
- finally {
- //stop our timer
- timer.stop();
- }
+ private List<LegacyQueueMessage> take(AsyncEventQueueType queueType) {
+ return take(queueType, false);
}
@@ -362,38 +375,15 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
}
-
- /**
- * calls the event handlers and returns a result with information on whether
- * it needs to be ack'd and whether it needs to be indexed
- * Ack message in SQS
- */
- public void ackUtilityQueue(final List<LegacyQueueMessage> messages) {
- try{
- utilityQueue.commitMessages( messages );
- }catch(Exception e){
- throw new RuntimeException("Unable to ack messages", e);
+ public void ack(final List<LegacyQueueMessage> messages, AsyncEventQueueType queueType, boolean isDeadQueue) {
+ if (queueType == AsyncEventQueueType.REGULAR && !isDeadQueue) {
+ // different functionality
+ ack(messages);
}
- }
-
- /**
- * ack messages in index dead letter queue
- */
- public void ackIndexDeadQueue(final List<LegacyQueueMessage> messages) {
- try{
- indexQueueDead.commitMessages( messages );
- }catch(Exception e){
- throw new RuntimeException("Unable to ack messages", e);
+ try {
+ getQueue(queueType, isDeadQueue).commitMessages( messages );
}
- }
-
- /**
- * ack messages in utility dead letter queue
- */
- public void ackUtilityDeadQueue(final List<LegacyQueueMessage> messages) {
- try{
- utilityQueueDead.commitMessages( messages );
- }catch(Exception e){
+ catch (Exception e) {
throw new RuntimeException("Unable to ack messages", e);
}
}
@@ -536,7 +526,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
applicationScope.getApplication().getUuid(), applicationScope.getApplication().getType());
offerTopic( new InitializeApplicationIndexEvent( queueFig.getPrimaryRegion(),
- new ReplicatedIndexLocationStrategy( indexLocationStrategy ) ), false);
+ new ReplicatedIndexLocationStrategy( indexLocationStrategy ) ), AsyncEventQueueType.REGULAR);
}
@@ -626,7 +616,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
edge.getType(), edge.getTargetNode().getUuid(), edge.getTargetNode().getType());
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
- offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ) );
+ offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ), AsyncEventQueueType.DELETE );
}
private IndexOperationMessage handleEdgeDelete(final LegacyQueueMessage message) {
@@ -660,9 +650,9 @@ public class AsyncEventServiceImpl implements AsyncEventService {
/**
* Queue up an indexOperationMessage for multi region execution
* @param indexOperationMessage
- * @param forUtilityQueue
+ * @param queueType
*/
- public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage, boolean forUtilityQueue) {
+ public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage, AsyncEventQueueType queueType) {
// don't try to produce something with nothing
if(indexOperationMessage == null || indexOperationMessage.isEmpty()){
@@ -688,7 +678,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
logger.trace("Offering ElasticsearchIndexEvent for message {}", newMessageId );
- offerTopic( elasticsearchIndexEvent, forUtilityQueue );
+ offerTopic( elasticsearchIndexEvent, queueType );
}
private void handleIndexOperation(final ElasticsearchIndexEvent elasticsearchIndexEvent)
@@ -764,7 +754,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
applicationScope.getApplication().getUuid(), entityId.getUuid(), entityId.getType());
offerTopic( new DeIndexOldVersionsEvent( queueFig.getPrimaryRegion(),
- new EntityIdScope( applicationScope, entityId), markedVersion), false);
+ new EntityIdScope( applicationScope, entityId), markedVersion), AsyncEventQueueType.DELETE );
}
@@ -824,7 +814,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
logger.trace("Offering EntityDeleteEvent for {}:{}", entityId.getUuid(), entityId.getType());
// sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
- offer( new EntityDeleteEvent(queueFig.getPrimaryRegion(), new EntityIdScope( applicationScope, entityId ) ) );
+ offer( new EntityDeleteEvent(queueFig.getPrimaryRegion(), new EntityIdScope( applicationScope, entityId ) ),
+ AsyncEventQueueType.DELETE );
}
private IndexOperationMessage handleEntityDelete(final LegacyQueueMessage message) {
@@ -879,25 +870,39 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public void start() {
final int indexCount = indexProcessorFig.getWorkerCount();
final int utilityCount = indexProcessorFig.getWorkerCountUtility();
+ final int deleteCount = indexProcessorFig.getWorkerCountDelete();
final int indexDeadCount = indexProcessorFig.getWorkerCountDeadLetter();
final int utilityDeadCount = indexProcessorFig.getWorkerCountUtilityDeadLetter();
- logger.info("Starting queue workers for indexing: index={} indexDLQ={} utility={} utilityDLQ={}", indexCount, indexDeadCount,
- utilityCount, utilityDeadCount);
+ final int deleteDeadCount = indexProcessorFig.getWorkerCountDeleteDeadLetter();
+ logger.info("Starting queue workers for indexing: index={} indexDLQ={} utility={} utilityDLQ={} delete={} deleteDLQ={}",
+ indexCount, indexDeadCount, utilityCount, utilityDeadCount, deleteCount, deleteDeadCount);
for (int i = 0; i < indexCount; i++) {
- startWorker(QUEUE_NAME);
+ startWorker(AsyncEventQueueType.REGULAR);
}
for (int i = 0; i < utilityCount; i++) {
- startWorker(QUEUE_NAME_UTILITY);
+ startWorker(AsyncEventQueueType.UTILITY);
}
- for (int i = 0; i < indexDeadCount; i++) {
- startDeadQueueWorker(QUEUE_NAME);
+ for (int i = 0; i < deleteCount; i++) {
+ startWorker(AsyncEventQueueType.DELETE);
}
+ if( indexQueue instanceof SNSQueueManagerImpl) {
+ logger.info("Queue manager implementation supports dead letters, start dead letter queue workers.");
+ for (int i = 0; i < indexDeadCount; i++) {
+ startDeadQueueWorker(AsyncEventQueueType.REGULAR);
+ }
- for (int i = 0; i < utilityDeadCount; i++) {
- startDeadQueueWorker(QUEUE_NAME_UTILITY);
+ for (int i = 0; i < utilityDeadCount; i++) {
+ startDeadQueueWorker(AsyncEventQueueType.UTILITY);
+ }
+
+ for (int i = 0; i < deleteDeadCount; i++) {
+ startDeadQueueWorker(AsyncEventQueueType.DELETE);
+ }
+ }else{
+ logger.info("Queue manager implementation does NOT support dead letters, NOT starting dead letter queue worker.");
}
}
@@ -916,11 +921,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
- private void startWorker(final String type) {
- Preconditions.checkNotNull(type, "Worker type required");
+ private void startWorker(final AsyncEventQueueType queueType) {
synchronized (mutex) {
- boolean isUtilityQueue = isNotEmpty(type) && type.toLowerCase().contains(QUEUE_NAME_UTILITY.toLowerCase());
+ String type = getQueueName(queueType);
Observable<List<LegacyQueueMessage>> consumer =
Observable.create( new Observable.OnSubscribe<List<LegacyQueueMessage>>() {
@@ -928,20 +932,15 @@ public class AsyncEventServiceImpl implements AsyncEventService {
public void call( final Subscriber<? super List<LegacyQueueMessage>> subscriber ) {
//name our thread so it's easy to see
- long threadNum = isUtilityQueue ?
- counterUtility.incrementAndGet() : counter.incrementAndGet();
- Thread.currentThread().setName( "QueueConsumer_" + type+ "_" + threadNum );
+ long threadNum = getCounter(queueType, false).incrementAndGet();
+ Thread.currentThread().setName( "QueueConsumer_" + type + "_" + threadNum );
List<LegacyQueueMessage> drainList = null;
do {
try {
- if ( isUtilityQueue ){
- drainList = takeFromUtilityQueue();
- }else{
- drainList = take();
+ drainList = take(queueType);
- }
//emit our list in it's entity to hand off to a worker pool
subscriber.onNext(drainList);
@@ -997,7 +996,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// submit the processed messages to index producer
List<LegacyQueueMessage> messagesToAck =
- submitToIndex( indexEventResults, isUtilityQueue );
+ submitToIndex( indexEventResults, queueType );
if ( messagesToAck.size() < messages.size() ) {
logger.warn(
@@ -1007,12 +1006,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// ack each message if making it to this point
if( messagesToAck.size() > 0 ){
-
- if ( isUtilityQueue ){
- ackUtilityQueue( messagesToAck );
- }else{
- ack( messagesToAck );
- }
+ ack(messagesToAck, queueType, false);
}
return messagesToAck;
@@ -1036,129 +1030,112 @@ public class AsyncEventServiceImpl implements AsyncEventService {
}
- private void startDeadQueueWorker(final String type) {
- Preconditions.checkNotNull(type, "Worker type required");
+ private void startDeadQueueWorker(final AsyncEventQueueType queueType) {
+ String type = getQueueName(queueType);
synchronized (mutex) {
- boolean isUtilityDeadQueue = isNotEmpty(type) && type.toLowerCase().contains(QUEUE_NAME_UTILITY.toLowerCase());
-
Observable<List<LegacyQueueMessage>> consumer =
- Observable.create( new Observable.OnSubscribe<List<LegacyQueueMessage>>() {
- @Override
- public void call( final Subscriber<? super List<LegacyQueueMessage>> subscriber ) {
+ Observable.create( new Observable.OnSubscribe<List<LegacyQueueMessage>>() {
+ @Override
+ public void call( final Subscriber<? super List<LegacyQueueMessage>> subscriber ) {
- //name our thread so it's easy to see
- long threadNum = isUtilityDeadQueue ?
- counterUtilityDead.incrementAndGet() : counterIndexDead.incrementAndGet();
- Thread.currentThread().setName( "QueueDeadLetterConsumer_" + type+ "_" + threadNum );
+ //name our thread so it's easy to see
+ long threadNum = getCounter(queueType, true).incrementAndGet();
+ Thread.currentThread().setName( "QueueDeadLetterConsumer_" + type + "_" + threadNum );
- List<LegacyQueueMessage> drainList = null;
+ List<LegacyQueueMessage> drainList = null;
- do {
- try {
- if ( isUtilityDeadQueue ){
- drainList = takeFromUtilityDeadQueue();
- }else{
- drainList = takeFromIndexDeadQueue();
- }
- //emit our list in it's entity to hand off to a worker pool
- subscriber.onNext(drainList);
+ do {
+ try {
+ drainList = take(queueType, true);
- //take since we're in flight
- inFlight.addAndGet( drainList.size() );
+ //emit our list in it's entity to hand off to a worker pool
+ subscriber.onNext(drainList);
- } catch ( Throwable t ) {
+ //take since we're in flight
+ inFlight.addAndGet( drainList.size() );
- final long sleepTime = indexProcessorFig.getDeadLetterFailureRetryTime();
+ } catch ( Throwable t ) {
- // there might be an error here during tests, just clean the cache
- indexQueueDead.clearQueueNameCache();
+ final long sleepTime = indexProcessorFig.getDeadLetterFailureRetryTime();
- if ( t instanceof InvalidQueryException ) {
+ // there might be an error here during tests, just clean the cache
+ indexQueueDead.clearQueueNameCache();
- // don't fill up log with exceptions when keyspace and column
- // families are not ready during bootstrap/setup
- logger.warn( "Failed to dequeue dead letters due to '{}'. Sleeping for {} ms",
- t.getMessage(), sleepTime );
+ if ( t instanceof InvalidQueryException ) {
- } else {
- logger.error( "Failed to dequeue dead letters. Sleeping for {} ms", sleepTime, t);
- }
+ // don't fill up log with exceptions when keyspace and column
+ // families are not ready during bootstrap/setup
+ logger.warn( "Failed to dequeue dead letters due to '{}'. Sleeping for {} ms",
+ t.getMessage(), sleepTime );
- if ( drainList != null ) {
- inFlight.addAndGet( -1 * drainList.size() );
- }
+ } else {
+ logger.error( "Failed to dequeue dead letters. Sleeping for {} ms", sleepTime, t);
+ }
- try { Thread.sleep( sleepTime ); } catch ( InterruptedException ie ) {}
+ if ( drainList != null ) {
+ inFlight.addAndGet( -1 * drainList.size() );
}
+
+ try { Thread.sleep( sleepTime ); } catch ( InterruptedException ie ) {}
}
- while ( true );
}
- } ) //this won't block our read loop, just reads and proceeds
- .flatMap( sqsMessages -> {
-
- //do this on a different schedule, and introduce concurrency
- // with flatmap for faster processing
- return Observable.just( sqsMessages )
-
- .map( messages -> {
- if ( messages == null || messages.size() == 0 ) {
- // no messages came from the queue, move on
- return null;
- }
-
- try {
- // put the dead letter messages back in the appropriate queue
- LegacyQueueManager returnQueue = null;
- String queueType;
- if (isUtilityDeadQueue) {
- returnQueue = utilityQueue;
- queueType = "utility";
- } else {
- returnQueue = indexQueue;
- queueType = "index";
- }
- List<LegacyQueueMessage> successMessages = returnQueue.sendQueueMessages(messages);
- for (LegacyQueueMessage msg : successMessages) {
- logger.warn("Returning message to {} queue: type:{}, messageId:{} body: {}", queueType, msg.getType(), msg.getMessageId(), msg.getStringBody());
- }
- int unsuccessfulMessagesSize = messages.size() - successMessages.size();
- if (unsuccessfulMessagesSize > 0) {
- // some messages couldn't be sent to originating queue, log
- Set<String> successMessageIds = new HashSet<>();
- for (LegacyQueueMessage msg : successMessages) {
- String messageId = msg.getMessageId();
- if (successMessageIds.contains(messageId)) {
- logger.warn("Found duplicate messageId in returned messages: {}", messageId);
- } else {
- successMessageIds.add(messageId);
- }
- }
- for (LegacyQueueMessage msg : messages) {
- String messageId = msg.getMessageId();
- if (!successMessageIds.contains(messageId)) {
- logger.warn("Failed to return message to {} queue: type:{} messageId:{} body: {}", queueType, msg.getType(), messageId, msg.getStringBody());
- }
- }
- }
-
- if (isUtilityDeadQueue) {
- ackUtilityDeadQueue(successMessages);
- } else {
- ackIndexDeadQueue(successMessages);
- }
-
- return messages;
- }
- catch ( Exception e ) {
- logger.error( "Failed to ack messages", e );
- return null;
- //do not rethrow so we can process all of them
- }
- } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() );
-
- //end flatMap
- }, indexProcessorFig.getEventConcurrencyFactor() );
+ while ( true );
+ }
+ } ) //this won't block our read loop, just reads and proceeds
+ .flatMap( sqsMessages -> {
+
+ //do this on a different schedule, and introduce concurrency
+ // with flatmap for faster processing
+ return Observable.just( sqsMessages )
+
+ .map( messages -> {
+ if ( messages == null || messages.size() == 0 ) {
+ // no messages came from the queue, move on
+ return null;
+ }
+
+ try {
+ // put the dead letter messages back in the appropriate queue
+ LegacyQueueManager returnQueue = getQueue(queueType, false);
+
+ List<LegacyQueueMessage> successMessages = returnQueue.sendQueueMessages(messages);
+ for (LegacyQueueMessage msg : successMessages) {
+ logger.warn("Returning message to {} queue: type:{}, messageId:{} body: {}", queueType.toString(), msg.getType(), msg.getMessageId(), msg.getStringBody());
+ }
+ int unsuccessfulMessagesSize = messages.size() - successMessages.size();
+ if (unsuccessfulMessagesSize > 0) {
+ // some messages couldn't be sent to originating queue, log
+ Set<String> successMessageIds = new HashSet<>();
+ for (LegacyQueueMessage msg : successMessages) {
+ String messageId = msg.getMessageId();
+ if (successMessageIds.contains(messageId)) {
+ logger.warn("Found duplicate messageId in returned messages: {}", messageId);
+ } else {
+ successMessageIds.add(messageId);
+ }
+ }
+ for (LegacyQueueMessage msg : messages) {
+ String messageId = msg.getMessageId();
+ if (!successMessageIds.contains(messageId)) {
+ logger.warn("Failed to return message to {} queue: type:{} messageId:{} body: {}", queueType.toString(), msg.getType(), messageId, msg.getStringBody());
+ }
+ }
+ }
+
+ ack(successMessages, queueType, true);
+
+ return messages;
+ }
+ catch ( Exception e ) {
+ logger.error( "Failed to ack messages", e );
+ return null;
+ //do not rethrow so we can process all of them
+ }
+ } ).subscribeOn( rxTaskScheduler.getAsyncIOScheduler() );
+
+ //end flatMap
+ }, indexProcessorFig.getEventConcurrencyFactor() );
//start in the background
@@ -1172,7 +1149,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
* Submit results to index and return the queue messages to be ack'd
*
*/
- private List<LegacyQueueMessage> submitToIndex(List<IndexEventResult> indexEventResults, boolean forUtilityQueue) {
+ private List<LegacyQueueMessage> submitToIndex(List<IndexEventResult> indexEventResults, AsyncEventQueueType queueType) {
// if nothing came back then return empty list
if(indexEventResults==null){
@@ -1199,7 +1176,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
// collect into a list of QueueMessages that can be ack'd later
.collect(Collectors.toList());
- queueIndexOperationMessage(combined, forUtilityQueue);
+ queueIndexOperationMessage(combined, queueType);
return queueMessages;
}
@@ -1210,10 +1187,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
new EntityIndexOperation( applicationScope, id, updatedSince);
queueIndexOperationMessage(
- eventBuilder.buildEntityIndex( entityIndexOperation ).toBlocking().lastOrDefault(null), false);
+ eventBuilder.buildEntityIndex( entityIndexOperation ).toBlocking().lastOrDefault(null), AsyncEventQueueType.REGULAR );
}
- public void indexBatch(final List<EdgeScope> edges, final long updatedSince, boolean forUtilityQueue) {
+ public void indexBatch(final List<EdgeScope> edges, final long updatedSince, AsyncEventQueueType queueType) {
final List<EntityIndexEvent> batch = new ArrayList<>();
edges.forEach(e -> {
@@ -1226,7 +1203,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
logger.trace("Offering batch of EntityIndexEvent of size {}", batch.size());
- offerBatch( batch, forUtilityQueue );
+ offerBatch( batch, queueType );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
index 7eecf04..eb63056 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java
@@ -40,10 +40,14 @@ public interface IndexProcessorFig extends GuicyFig {
String ELASTICSEARCH_WORKER_COUNT_UTILITY = "elasticsearch.worker_count_utility";
+ String ELASTICSEARCH_WORKER_COUNT_DELETE = "elasticsearch.worker_count_delete";
+
String ELASTICSEARCH_WORKER_COUNT_DEADLETTER = "elasticsearch.worker_count_deadletter";
String ELASTICSEARCH_WORKER_COUNT_UTILITY_DEADLETTER = "elasticsearch.worker_count_utility_deadletter";
+ String ELASTICSEARCH_WORKER_COUNT_DELETE_DEADLETTER = "elasticsearch.worker_count_delete_deadletter";
+
String EVENT_CONCURRENCY_FACTOR = "event.concurrency.factor";
String ELASTICSEARCH_QUEUE_IMPL = "elasticsearch.queue_impl";
@@ -105,6 +109,13 @@ public interface IndexProcessorFig extends GuicyFig {
int getWorkerCountUtility();
/**
+ * The number of worker threads used to read delete requests from the queue.
+ */
+ @Default("1")
+ @Key(ELASTICSEARCH_WORKER_COUNT_DELETE)
+ int getWorkerCountDelete();
+
+ /**
* The number of worker threads used to read dead messages from the index dead letter queue and reload them into the index queue.
*/
@Default("1")
@@ -119,6 +130,13 @@ public interface IndexProcessorFig extends GuicyFig {
int getWorkerCountUtilityDeadLetter();
/**
+ * The number of worker threads used to read dead messages from the delete dead letter queue and reload them into the delete queue.
+ */
+ @Default("1")
+ @Key(ELASTICSEARCH_WORKER_COUNT_DELETE_DEADLETTER)
+ int getWorkerCountDeleteDeadLetter();
+
+ /**
* Set the implementation to use for queuing.
* Valid values: TEST, LOCAL, SQS, SNS
* NOTE: SQS and SNS equate to the same implementation of Amazon queue services.
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
index 2b3573e..d6bdd93 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.index;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
import org.apache.usergrid.corepersistence.rx.impl.EdgeScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -42,7 +43,7 @@ public interface ReIndexAction {
* Index a batch list of entities. Goes to the utility queue.
* @param edges
* @param updatedSince
- * @param forUtilityQueue
+ * @param queueType
*/
- void indexBatch(final List<EdgeScope> edges, final long updatedSince, boolean forUtilityQueue);
+ void indexBatch(final List<EdgeScope> edges, final long updatedSince, AsyncEventQueueType queueType);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 0660d5e..c7371b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -27,6 +27,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -169,7 +170,7 @@ public class ReIndexServiceImpl implements ReIndexService {
.buffer( indexProcessorFig.getReindexBufferSize())
.doOnNext( edgeScopes -> {
logger.info("Sending batch of {} to be indexed.", edgeScopes.size());
- indexService.indexBatch(edgeScopes, modifiedSince, true);
+ indexService.indexBatch(edgeScopes, modifiedSince, AsyncEventQueueType.UTILITY);
count.addAndGet(edgeScopes.size() );
if( edgeScopes.size() > 0 ) {
writeCursorState(jobId, edgeScopes.get(edgeScopes.size() - 1));
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
index 83f4c8b..4886b08 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
@@ -124,7 +125,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
indexMessageObservable
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -139,7 +140,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
entityDeleteResults.getIndexObservable()
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -159,7 +160,7 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
entityDeleteResults.getIndexObservable()
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -252,13 +253,13 @@ public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id,
}
}
- private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector() {
+ private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector(AsyncEventQueueType queueType) {
return observable -> observable
.collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
.filter(msg -> !msg.isEmpty())
.doOnNext(indexOperation -> {
- asyncEventService.queueIndexOperationMessage(indexOperation, false);
+ asyncEventService.queueIndexOperationMessage(indexOperation, queueType);
});
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/570e1ab4/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
index 1afb524..e8a20dd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadReverseGraphFilter.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+import org.apache.usergrid.corepersistence.asyncevents.AsyncEventQueueType;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilder;
import org.apache.usergrid.corepersistence.asyncevents.EventBuilderImpl;
@@ -124,7 +125,7 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
final Observable<IndexOperationMessage> indexMessageObservable = eventBuilder.buildDeleteEdge(applicationScope, markedEdge);
indexMessageObservable
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -139,7 +140,7 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, sourceNodeId);
entityDeleteResults.getIndexObservable()
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -159,7 +160,7 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
entityDeleteResults = eventBuilder.buildEntityDelete(applicationScope, targetNodeId);
entityDeleteResults.getIndexObservable()
- .compose(applyCollector())
+ .compose(applyCollector(AsyncEventQueueType.DELETE))
.subscribeOn(rxTaskScheduler.getAsyncIOScheduler())
.subscribe();
@@ -245,13 +246,13 @@ public abstract class AbstractReadReverseGraphFilter extends AbstractPathFilter<
}
}
- private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector() {
+ private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector(AsyncEventQueueType queueType) {
return observable -> observable
.collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single))
.filter(msg -> !msg.isEmpty())
.doOnNext(indexOperation -> {
- asyncEventService.queueIndexOperationMessage(indexOperation, false);
+ asyncEventService.queueIndexOperationMessage(indexOperation, queueType);
});
}
[18/27] usergrid git commit: Merge branch 'collectionClearJob'
Posted by mr...@apache.org.
Merge branch 'collectionClearJob'
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8e9eff8d
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8e9eff8d
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8e9eff8d
Branch: refs/heads/expose-reindex
Commit: 8e9eff8d9340fc85b7d4b544fa542e388974525a
Parents: 36e28a8 11823f2
Author: Mike Dunker <md...@google.com>
Authored: Mon Aug 28 16:23:01 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Aug 28 16:23:01 2017 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +-
.../usergrid/corepersistence/CoreModule.java | 1 +
.../corepersistence/CpEntityManager.java | 3 +-
.../corepersistence/CpEntityManagerFactory.java | 2 +-
.../corepersistence/EntityManagerFig.java | 2 +-
.../asyncevents/AsyncEventQueueType.java | 35 ++
.../asyncevents/AsyncEventService.java | 7 +-
.../asyncevents/AsyncEventServiceImpl.java | 545 +++++++++----------
.../asyncevents/EventBuilder.java | 26 +-
.../asyncevents/EventBuilderImpl.java | 205 +++++--
.../asyncevents/model/EntityDeleteEvent.java | 27 +-
.../index/CollectionDeleteAction.java | 43 ++
.../index/CollectionDeleteRequestBuilder.java | 92 ++++
.../CollectionDeleteRequestBuilderImpl.java | 146 +++++
.../index/CollectionDeleteService.java | 108 ++++
.../index/CollectionDeleteServiceImpl.java | 299 ++++++++++
.../index/CollectionSettingsImpl.java | 1 -
.../index/IndexProcessorFig.java | 27 +
.../corepersistence/index/IndexService.java | 16 +-
.../corepersistence/index/IndexServiceImpl.java | 56 +-
.../corepersistence/index/ReIndexAction.java | 5 +-
.../index/ReIndexServiceImpl.java | 5 +-
.../read/traverse/AbstractReadGraphFilter.java | 42 +-
.../AbstractReadReverseGraphFilter.java | 41 +-
.../usergrid/persistence/EntityManager.java | 2 +-
.../org/apache/usergrid/persistence/Query.java | 17 +
.../persistence/entities/Application.java | 26 +
.../rx/EdgesFromSourceObservableIT.java | 2 +-
.../rx/EdgesToTargetObservableIT.java | 4 +-
.../persistence/CollectionDeleteTest.java | 266 +++++++++
.../usergrid/persistence/CoreSchemaManager.java | 16 +-
.../usergrid/persistence/RebuildIndexTest.java | 36 +-
.../core/src/test/resources/project.properties | 2 +-
.../resources/usergrid-custom-test.properties | 3 +-
.../actorsystem/ActorSystemManagerImpl.java | 3 +-
.../src/test/resources/dynamic-test.properties | 1 +
.../src/test/resources/usergrid-CHOP.properties | 1 +
.../src/test/resources/usergrid-UNIT.properties | 1 +
.../src/test/resources/usergrid.properties | 1 +
.../persistence/graph/GraphManager.java | 2 +-
.../graph/impl/GraphManagerImpl.java | 21 +-
.../graph/impl/SimpleSearchByEdgeType.java | 9 +
.../graph/impl/stage/NodeDeleteListener.java | 6 +-
.../impl/stage/NodeDeleteListenerImpl.java | 64 +--
.../graph/serialization/EdgesObservable.java | 6 +-
.../impl/EdgeMetadataSerializationV2Impl.java | 1 -
.../serialization/impl/EdgesObservableImpl.java | 22 +-
.../impl/TargetIdObservableImpl.java | 4 +-
.../impl/migration/EdgeDataMigrationImpl.java | 2 +-
.../graph/impl/NodeDeleteListenerTest.java | 8 +-
.../src/test/resources/usergrid-AWS.properties | 1 +
.../src/test/resources/usergrid-CHOP.properties | 1 +
.../src/test/resources/usergrid-UNIT.properties | 1 +
.../src/test/resources/usergrid.properties | 1 +
.../usergrid/persistence/index/EntityIndex.java | 12 +-
.../index/impl/EsEntityIndexImpl.java | 65 +--
.../persistence/index/query/Identifier.java | 3 +-
.../src/test/resources/dynamic-test.properties | 1 +
.../src/test/resources/usergrid-CHOP.properties | 1 +
.../src/test/resources/usergrid-UNIT.properties | 1 +
stack/corepersistence/queue/pom.xml | 18 +-
.../queues/impl/QueueSerializationImpl.java | 2 +-
.../persistence/queue/guice/QueueModule.java | 2 +-
.../persistence/qakka/KeyspaceDropper.java | 6 +-
.../qakka/core/QueueMessageManagerTest.java | 2 +-
.../queue/src/test/resources/qakka.properties | 1 +
stack/pom.xml | 8 +-
.../resources/usergrid-custom-test.properties | 2 +
stack/rest/pom.xml | 4 +-
.../rest/applications/CollectionResource.java | 130 ++++-
.../rest/applications/users/UserResource.java | 9 +
.../rest/management/users/UserResource.java | 16 +
.../apache/usergrid/rest/TestResource/error.jsp | 4 +-
.../apache/usergrid/rest/TestResource/test.jsp | 5 +-
.../ApplicationResource/authorize_form.jsp | 15 +-
.../applications/ApplicationResource/error.jsp | 4 +-
.../users/UserResource/activate.jsp | 4 +-
.../applications/users/UserResource/confirm.jsp | 4 +-
.../applications/users/UserResource/error.jsp | 4 +-
.../users/UserResource/resetpw_email_form.jsp | 7 +-
.../UserResource/resetpw_email_success.jsp | 4 +-
.../users/UserResource/resetpw_set_form.jsp | 9 +-
.../users/UserResource/resetpw_set_success.jsp | 4 +-
.../applications/users/UsersResource/error.jsp | 4 +-
.../users/UsersResource/resetpw_email_form.jsp | 5 +-
.../UsersResource/resetpw_email_success.jsp | 4 +-
.../ManagementResource/authorize_form.jsp | 15 +-
.../management/ManagementResource/error.jsp | 4 +-
.../OrganizationResource/activate.jsp | 4 +-
.../OrganizationResource/confirm.jsp | 4 +-
.../OrganizationResource/error.jsp | 4 +-
.../management/users/UserResource/activate.jsp | 4 +-
.../management/users/UserResource/confirm.jsp | 4 +-
.../management/users/UserResource/error.jsp | 4 +-
.../users/UserResource/resetpw_email_form.jsp | 5 +-
.../UserResource/resetpw_email_success.jsp | 4 +-
.../users/UserResource/resetpw_set_form.jsp | 8 +-
.../users/UserResource/resetpw_set_success.jsp | 4 +-
.../management/users/UsersResource/error.jsp | 4 +-
.../users/UsersResource/resetpw_email_form.jsp | 4 +-
.../UsersResource/resetpw_email_success.jsp | 4 +-
.../rest/src/test/resources/project.properties | 2 +-
.../resources/usergrid-custom-test.properties | 4 +-
stack/services/pom.xml | 4 +-
.../usergrid/management/ManagementService.java | 10 +-
.../cassandra/ManagementServiceImpl.java | 10 +
.../usergrid/services/ActivitiesServiceIT.java | 9 +-
.../usergrid/services/ServiceInvocationIT.java | 3 +
.../AbstractServiceNotificationIT.java | 4 +-
.../gcm/NotificationsServiceIT.java | 2 +-
.../src/test/resources/project.properties | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
stack/test-utils/pom.xml | 4 +-
.../src/test/resources/project.properties | 2 +-
stack/tools/pom.xml | 2 +-
stack/websocket/pom.xml | 2 +-
116 files changed, 2013 insertions(+), 752 deletions(-)
----------------------------------------------------------------------
[24/27] usergrid git commit: add reset_url as field in admin
confirmation email
Posted by mr...@apache.org.
add reset_url as field in admin confirmation email
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/173be8a4
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/173be8a4
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/173be8a4
Branch: refs/heads/expose-reindex
Commit: 173be8a4b968eda617f73a3baf05fa7870594657
Parents: 7f40d68
Author: Mike Dunker <md...@google.com>
Authored: Fri Sep 22 14:51:55 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Tue Sep 26 08:09:04 2017 -0700
----------------------------------------------------------------------
.../management/cassandra/ManagementServiceImpl.java | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/173be8a4/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
index 2302d8d..ab93563 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/cassandra/ManagementServiceImpl.java
@@ -2680,10 +2680,17 @@ public class ManagementServiceImpl implements ManagementService {
String token = getConfirmationTokenForAdminUser(user.getUuid(), 0, organizationId);
OrganizationConfig orgConfig = organizationId != null ?
getOrganizationConfigByUuid(organizationId) : getOrganizationConfigForUserInfo(user);
- String confirmation_url = orgConfig.getFullUrl(WorkflowUrl.ADMIN_CONFIRMATION_URL,
- user.getUuid().toString()) + "?token=" + token;
+ String confirmation_url = orgConfig.getFullUrl(WorkflowUrl.ADMIN_CONFIRMATION_URL, user.getUuid().toString()) +
+ "?token=" + token;
+
+ String reset_token = getPasswordResetTokenForAdminUser( user.getUuid(), 0, organizationId );
+ String resetPropertyUrl = orgConfig.getFullUrlTemplate(WorkflowUrl.ADMIN_RESETPW_URL);
+ String reset_url = String.format(resetPropertyUrl, user.getUuid().toString())
+ + "?token=" + reset_token;
+
sendAdminUserEmail(user, "User Account Confirmation: " + user.getEmail(),
- emailMsg(hashMap("confirm_email", user.getEmail()).map("confirmation_url", confirmation_url),
+ emailMsg(hashMap("confirm_email", user.getEmail()).map("confirmation_url", confirmation_url)
+ .map("reset_url", reset_url),
PROPERTIES_EMAIL_ADMIN_CONFIRMATION));
}