You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by md...@apache.org on 2017/11/20 17:55:56 UTC
[1/3] usergrid git commit: add "replace_all" to POST settings to
allow merging of setting properties Modified unit tests to allow for a more
flexible response times [Forced Update!]
Repository: usergrid
Updated Branches:
refs/heads/3.0.0_experimental 697b291ec -> 5465d807e (forced update)
add "replace_all" to POST settings to allow merging of setting properties
Modified unit tests to allow for a more flexible response times
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2110bfbd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2110bfbd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2110bfbd
Branch: refs/heads/3.0.0_experimental
Commit: 2110bfbd44f0b3dd8fcc9ec7cdf9c8c287428ff6
Parents: 8b945c6
Author: Peter Johnson <pj...@apigee.com>
Authored: Thu Nov 16 13:17:33 2017 -0800
Committer: Peter Johnson <pj...@apigee.com>
Committed: Thu Nov 16 16:37:13 2017 -0800
----------------------------------------------------------------------
.../persistence/CollectionDeleteTest.java | 33 +-
.../usergrid/persistence/RebuildIndexTest.java | 349 ++++++++++---------
.../rest/applications/CollectionResource.java | 38 +-
.../collection/CollectionsResourceIT.java | 83 ++++-
.../collection/users/UserResourceIT.java | 33 +-
.../resource/endpoints/CollectionEndpoint.java | 30 +-
.../AbstractServiceNotificationIT.java | 10 +-
.../apns/NotificationsServiceIT.java | 16 +-
.../gcm/NotificationsServiceIT.java | 9 +-
9 files changed, 396 insertions(+), 205 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
index 42afa67..9e5598c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CollectionDeleteTest.java
@@ -130,9 +130,7 @@ public class CollectionDeleteTest extends AbstractCoreIT {
}
entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
- if ( i % 10 == 0 ) {
- logger.info( "Created {} entities after delete time", i );
- }
+ logger.info( "Created {} entities after delete time created time {} ", i , entity.getCreated());
}
logger.info("Created {} entities after delete time", ENTITIES_TO_ADD_AFTER_TIME);
@@ -155,11 +153,11 @@ public class CollectionDeleteTest extends AbstractCoreIT {
waitForDelete( status, collectionDeleteService );
- app.waitForQueueDrainAndRefreshIndex(15000);
+ //app.waitForQueueDrainAndRefreshIndex(15000);
// ----------------- test that we can read the entries after the timestamp
- readData( em, collectionName,ENTITIES_TO_ADD_AFTER_TIME);
+ retryReadData( em, collectionName, ENTITIES_TO_ADD_AFTER_TIME, 60);
}
/**
@@ -199,6 +197,19 @@ public class CollectionDeleteTest extends AbstractCoreIT {
}
}
+ private int retryReadData(EntityManager em, String collectionName, int expectedEntities, int retry) throws Exception {
+ int count = -1;
+ do {
+ try {
+ count = readData(em, collectionName, expectedEntities);
+ } catch (Exception ignore) {
+ logger.info( "caught exception ", ignore);
+ }
+ logger.info( "read {} expected {}" , count, expectedEntities);
+ } while (count != expectedEntities && --retry >=0);
+ assertEquals( "Did not get expected entities", expectedEntities, count );
+ return count;
+ }
private int readData(EntityManager em, String collectionName, int expectedEntities)
throws Exception {
@@ -209,6 +220,7 @@ public class CollectionDeleteTest extends AbstractCoreIT {
Query.Level.ALL_PROPERTIES, false);
int count = 0;
+ List<Entity> list = new ArrayList<>();
while ( true ) {
if (results.getEntities().size() == 0) {
@@ -225,6 +237,7 @@ public class CollectionDeleteTest extends AbstractCoreIT {
}
lastEntityUUID = e.getUuid();
count++;
+ list.add(e);
}
results = em.getCollection(em.getApplicationRef(), collectionName, lastEntityUUID, expectedEntities,
@@ -233,6 +246,16 @@ public class CollectionDeleteTest extends AbstractCoreIT {
}
logger.info("read {} total entities", count);
+ if (count != expectedEntities) {
+ logger.info("Expected {} did not match actual {}", expectedEntities, count);
+ if (count < 20) {
+ for (Entity e : list) {
+ Object key = e.getProperty("key2");
+ logger.info("Entity key {} ceated {}", key, e.getCreated());
+ }
+ }
+ }
+
assertEquals( "Did not get expected entities", expectedEntities, count );
return count;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index a1ff84b..57962c0 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -51,7 +51,7 @@ import static org.junit.Assert.fail;
@NotThreadSafe
public class RebuildIndexTest extends AbstractCoreIT {
- private static final Logger logger = LoggerFactory.getLogger( RebuildIndexTest.class );
+ private static final Logger logger = LoggerFactory.getLogger(RebuildIndexTest.class);
private static final MetricRegistry registry = new MetricRegistry();
@@ -70,183 +70,182 @@ public class RebuildIndexTest extends AbstractCoreIT {
@After
public void printReport() {
- logger.debug( "Printing metrics report" );
+ logger.debug("Printing metrics report");
}
- @Test( timeout = 240000 )
+ @Test(timeout = 240000)
public void rebuildOneCollectionIndex() throws Exception {
- logger.info( "Started rebuildOneCollectionIndex()" );
+ logger.info("Started rebuildOneCollectionIndex()");
- String rand = RandomStringUtils.randomAlphanumeric( 5 );
- final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
+ String rand = RandomStringUtils.randomAlphanumeric(5);
+ final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
- final EntityManager em = setup.getEmf().getEntityManager( appId );
+ final EntityManager em = setup.getEmf().getEntityManager(appId);
- final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class );
+ final ReIndexService reIndexService = setup.getInjector().getInstance(ReIndexService.class);
// ----------------- create a bunch of entities
Map<String, Object> entityMap = new HashMap<String, Object>() {{
- put( "key1", 1000 );
- put( "key2", 2000 );
- put( "key3", "Some value" );
+ put("key1", 1000);
+ put("key2", 2000);
+ put("key3", "Some value");
}};
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
int herderCount = 0;
int shepardCount = 0;
- for ( int i = 0; i < ENTITIES_TO_INDEX; i++ ) {
+ for (int i = 0; i < ENTITIES_TO_INDEX; i++) {
final Entity entity;
try {
- entityMap.put( "key", i );
+ entityMap.put("key", i);
- if ( i % 2 == 0 ) {
- entity = em.create( "catherder", entityMap );
+ if (i % 2 == 0) {
+ entity = em.create("catherder", entityMap);
herderCount++;
- }
- else {
- entity = em.create( "catshepard", entityMap );
+ } else {
+ entity = em.create("catshepard", entityMap);
shepardCount++;
}
- }
- catch ( Exception ex ) {
- throw new RuntimeException( "Error creating entity", ex );
+ } catch (Exception ex) {
+ throw new RuntimeException("Error creating entity", ex);
}
- entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
- if ( i % 10 == 0 ) {
- logger.info( "Created {} entities", i );
+ entityRefs.add(new SimpleEntityRef(entity.getType(), entity.getUuid()));
+ if (i % 10 == 0) {
+ logger.info("Created {} entities", i);
}
}
- logger.info( "Created {} entities", ENTITIES_TO_INDEX );
- app.waitForQueueDrainAndRefreshIndex(5000);
+ logger.info("Created {} entities", ENTITIES_TO_INDEX);
+ app.waitForQueueDrainAndRefreshIndex(1000);
// ----------------- test that we can read them, should work fine
- logger.debug( "Read the data" );
- readData( em, "catherders", herderCount, 0 );
- readData( em, "catshepards", shepardCount, 0 );
+ logger.debug("Read the data");
+ retryReadData(em, "catherders", herderCount, 0, 10);
+ retryReadData(em, "catshepards", shepardCount, 0, 10);
// ----------------- delete the system and application indexes
- logger.debug( "Deleting apps" );
- deleteIndex( em.getApplicationId() );
+ logger.debug("Deleting apps");
+ deleteIndex(em.getApplicationId());
// ----------------- test that we can read them, should fail
- logger.debug( "Reading data, should fail this time " );
+ logger.debug("Reading data, should fail this time ");
//should be no data
- readData( em, "testTypes", 0, 0 );
+ readData(em, "testTypes", 0, 0);
// ----------------- rebuild index for catherders only
- logger.debug( "Preparing to rebuild all indexes" );
+ logger.debug("Preparing to rebuild all indexes");
final ReIndexRequestBuilder builder =
- reIndexService.getBuilder().withApplicationId( em.getApplicationId() ).withCollection( "catherders" );
+ reIndexService.getBuilder().withApplicationId(em.getApplicationId()).withCollection("catherders");
- ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder );
+ ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder);
- assertNotNull( status.getJobId(), "JobId is present" );
+ assertNotNull(status.getJobId(), "JobId is present");
- logger.info( "Rebuilt index" );
+ logger.info("Rebuilt index");
- waitForRebuild( status, reIndexService );
+ waitForRebuild(status, reIndexService);
- app.waitForQueueDrainAndRefreshIndex(15000);
+ //app.waitForQueueDrainAndRefreshIndex(15000);
// ----------------- test that we can read the catherder collection and not the catshepard
- readData( em, "catherders", herderCount, 0 );
- readData( em, "catshepards", 0, 0 );
+ retryReadData(em, "catherders", herderCount, 0, 30);
+ retryReadData(em, "catshepards", 0, 0, 30);
}
- @Test( timeout = 240000 )
+ @Test(timeout = 240000)
public void rebuildIndex() throws Exception {
- logger.info( "Started rebuildIndex()" );
+ logger.info("Started rebuildIndex()");
- String rand = RandomStringUtils.randomAlphanumeric( 5 );
- final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
+ String rand = RandomStringUtils.randomAlphanumeric(5);
+ final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
- final EntityManager em = setup.getEmf().getEntityManager( appId );
+ final EntityManager em = setup.getEmf().getEntityManager(appId);
- final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class );
+ final ReIndexService reIndexService = setup.getInjector().getInstance(ReIndexService.class);
// ----------------- create a bunch of entities
Map<String, Object> entityMap = new HashMap<String, Object>() {{
- put( "key1", 1000 );
- put( "key2", 2000 );
- put( "key3", "Some value" );
+ put("key1", 1000);
+ put("key2", 2000);
+ put("key3", "Some value");
}};
Map<String, Object> cat1map = new HashMap<String, Object>() {{
- put( "name", "enzo" );
- put( "color", "orange" );
+ put("name", "enzo");
+ put("color", "orange");
}};
Map<String, Object> cat2map = new HashMap<String, Object>() {{
- put( "name", "marquee" );
- put( "color", "grey" );
+ put("name", "marquee");
+ put("color", "grey");
}};
Map<String, Object> cat3map = new HashMap<String, Object>() {{
- put( "name", "bertha" );
- put( "color", "tabby" );
+ put("name", "bertha");
+ put("color", "tabby");
}};
- Entity cat1 = em.create( "cat", cat1map );
- Entity cat2 = em.create( "cat", cat2map );
- Entity cat3 = em.create( "cat", cat3map );
+ Entity cat1 = em.create("cat", cat1map);
+ Entity cat2 = em.create("cat", cat2map);
+ Entity cat3 = em.create("cat", cat3map);
List<EntityRef> entityRefs = new ArrayList<>();
- for ( int i = 0; i < ENTITIES_TO_INDEX; i++ ) {
+ for (int i = 0; i < ENTITIES_TO_INDEX; i++) {
final Entity entity;
try {
- entityMap.put( "key", i );
- entity = em.create( "testType", entityMap );
-
-
- em.createConnection( entity, "herds", cat1 );
- em.createConnection( entity, "herds", cat2 );
- em.createConnection( entity, "herds", cat3 );
- }
- catch ( Exception ex ) {
- throw new RuntimeException( "Error creating entity", ex );
+ entityMap.put("key", i);
+ entity = em.create("testType", entityMap);
+
+ em.createConnection(entity, "herds", cat1);
+ em.createConnection(entity, "herds", cat2);
+ em.createConnection(entity, "herds", cat3);
+ } catch (Exception ex) {
+ throw new RuntimeException("Error creating entity", ex);
}
- entityRefs.add( new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
- if ( i % 10 == 0 ) {
- logger.info( "Created {} entities", i );
+ entityRefs.add(new SimpleEntityRef(entity.getType(), entity.getUuid()));
+ if (i % 10 == 0) {
+ logger.info("Created {} entities", i);
}
}
- logger.info( "Created {} entities", ENTITIES_TO_INDEX );
- app.waitForQueueDrainAndRefreshIndex(30000);
+ logger.info("Created {} entities", ENTITIES_TO_INDEX);
+ //app.waitForQueueDrainAndRefreshIndex(30000);
// ----------------- test that we can read them, should work fine
- logger.debug( "Read the data" );
+ logger.debug("Read the data");
final String collectionName = "testtypes";
- readData( em, collectionName, ENTITIES_TO_INDEX, 3 );
+
+ retryReadData(em, collectionName, ENTITIES_TO_INDEX, 3, 20);
+
+ readData(em, collectionName, ENTITIES_TO_INDEX, 3);
// ----------------- delete the system and application indexes
- logger.debug( "Deleting app index" );
+ logger.debug("Deleting app index");
- deleteIndex( em.getApplicationId() );
+ deleteIndex(em.getApplicationId());
app.waitForQueueDrainAndRefreshIndex();
@@ -257,64 +256,62 @@ public class RebuildIndexTest extends AbstractCoreIT {
// ----------------- test that we can read them, should fail
- logger.debug( "Reading data, should fail this time " );
-
- readData( em, collectionName, 0, 0 );
+ logger.debug("Reading data, should fail this time ");
+ readData(em, collectionName, 0, 0);
// ----------------- rebuild index
- logger.debug( "Preparing to rebuild all indexes" );
+ logger.debug("Preparing to rebuild all indexes");
;
try {
final ReIndexRequestBuilder builder =
- reIndexService.getBuilder().withApplicationId( em.getApplicationId() );
+ reIndexService.getBuilder().withApplicationId(em.getApplicationId());
- ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder );
+ ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder);
- assertNotNull( status.getJobId(), "JobId is present" );
+ assertNotNull(status.getJobId(), "JobId is present");
- logger.info( "Rebuilt index, jobID={}", status.getJobId());
+ logger.info("Rebuilt index, jobID={}", status.getJobId());
- waitForRebuild( status, reIndexService );
+ waitForRebuild(status, reIndexService);
- logger.info( "Rebuilt index" );
+ logger.info("Rebuilt index");
- }
- catch ( Exception ex ) {
- logger.error( "Error rebuilding index", ex );
+ } catch (Exception ex) {
+ logger.error("Error rebuilding index", ex);
fail();
}
// ----------------- test that we can read them
app.waitForQueueDrainAndRefreshIndex(15000);
- readData( em, collectionName, ENTITIES_TO_INDEX, 3 );
+ readData(em, collectionName, ENTITIES_TO_INDEX, 3);
}
- @Test( timeout = 120000 )
+ @Test(timeout = 120000)
public void rebuildIndexGeo() throws Exception {
- logger.info( "Started rebuildIndexGeo()" );
+ logger.info("Started rebuildIndexGeo()");
- String rand = RandomStringUtils.randomAlphanumeric( 5 );
- final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
+ String rand = RandomStringUtils.randomAlphanumeric(5);
+ final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
- final EntityManager em = setup.getEmf().getEntityManager( appId );
+ final EntityManager em = setup.getEmf().getEntityManager(appId);
- final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class );
+ final ReIndexService reIndexService = setup.getInjector().getInstance(ReIndexService.class);
// ----------------- create a bunch of entities
Map<String, Object> cat1map = new HashMap<String, Object>() {{
- put( "name", "enzo" );
- put( "color", "grey" );
+ put("name", "enzo");
+ put("color", "grey");
put("location", new LinkedHashMap<String, Object>() {{
put("latitude", -35.746369);
put("longitude", 150.952183);
@@ -323,44 +320,44 @@ public class RebuildIndexTest extends AbstractCoreIT {
final double lat = -34.746369;
final double lon = 152.952183;
Map<String, Object> cat2map = new HashMap<String, Object>() {{
- put( "name", "marquee" );
- put( "color", "grey" );
+ put("name", "marquee");
+ put("color", "grey");
put("location", new LinkedHashMap<String, Object>() {{
put("latitude", lat);
put("longitude", lon);
}});
}};
Map<String, Object> cat3map = new HashMap<String, Object>() {{
- put( "name", "bertha" );
- put( "color", "grey" );
+ put("name", "bertha");
+ put("color", "grey");
put("location", new LinkedHashMap<String, Object>() {{
put("latitude", -33.746369);
put("longitude", 150.952183);
}});
}};
- Entity cat1 = em.create( "cat", cat1map );
- Entity cat2 = em.create( "cat", cat2map );
- Entity cat3 = em.create( "cat", cat3map );
+ Entity cat1 = em.create("cat", cat1map);
+ Entity cat2 = em.create("cat", cat2map);
+ Entity cat3 = em.create("cat", cat3map);
- logger.info( "Created {} entities", ENTITIES_TO_INDEX );
+ logger.info("Created {} entities", ENTITIES_TO_INDEX);
app.waitForQueueDrainAndRefreshIndex(5000);
// ----------------- test that we can read them, should work fine
- logger.debug( "Read the data" );
+ logger.debug("Read the data");
final String collectionName = "cats";
- Query q = Query.fromQL( "select * where color='grey'" ).withLimit( 1000 );
- Results results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 3 );
- assertEquals(3,results.size());
+ Query q = Query.fromQL("select * where color='grey'").withLimit(1000);
+ Results results = em.searchCollectionConsistent(em.getApplicationRef(), collectionName, q, 3);
+ assertEquals(3, results.size());
// ----------------- delete the system and application indexes
- logger.debug( "Deleting app index" );
+ logger.debug("Deleting app index");
- deleteIndex( em.getApplicationId() );
+ deleteIndex(em.getApplicationId());
// ----------------- test that we can read them, should fail
@@ -369,87 +366,86 @@ public class RebuildIndexTest extends AbstractCoreIT {
// ----------------- test that we can read them, should fail
- logger.debug( "Reading data, should fail this time " );
+ logger.debug("Reading data, should fail this time ");
- results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 0 );
- assertEquals(results.size(),0);
+ results = em.searchCollectionConsistent(em.getApplicationRef(), collectionName, q, 0);
+ assertEquals(results.size(), 0);
// ----------------- rebuild index
- logger.debug( "Preparing to rebuild all indexes" );
+ logger.debug("Preparing to rebuild all indexes");
try {
final ReIndexRequestBuilder builder =
- reIndexService.getBuilder().withApplicationId( em.getApplicationId() );
+ reIndexService.getBuilder().withApplicationId(em.getApplicationId());
- ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder );
+ ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder);
- assertNotNull( status.getJobId(), "JobId is present" );
+ assertNotNull(status.getJobId(), "JobId is present");
- logger.info( "Rebuilt index" );
+ logger.info("Rebuilt index");
- waitForRebuild( status, reIndexService );
+ waitForRebuild(status, reIndexService);
- logger.info( "Rebuilt index" );
+ logger.info("Rebuilt index");
- }
- catch ( Exception ex ) {
- logger.error( "Error rebuilding index", ex );
+ } catch (Exception ex) {
+ logger.error("Error rebuilding index", ex);
fail();
}
// ----------------- test that we can read them
app.waitForQueueDrainAndRefreshIndex(5000);
- results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 3 );
- assertEquals(results.size(),3);
- q = Query.fromQL("select * where location within 100 of "+lat+", "+lon);
- results = em.searchCollectionConsistent( em.getApplicationRef(), collectionName, q, 1 );
- assertEquals(results.size(),1);
+ results = em.searchCollectionConsistent(em.getApplicationRef(), collectionName, q, 3);
+ assertEquals(results.size(), 3);
+ q = Query.fromQL("select * where location within 100 of " + lat + ", " + lon);
+ results = em.searchCollectionConsistent(em.getApplicationRef(), collectionName, q, 1);
+ assertEquals(results.size(), 1);
}
- @Test( timeout = 120000 )
+ @Test(timeout = 120000)
public void rebuildUpdatedSince() throws Exception {
- logger.info( "Started rebuildUpdatedSince()" );
+ logger.info("Started rebuildUpdatedSince()");
- String rand = RandomStringUtils.randomAlphanumeric( 5 );
- final UUID appId = setup.createApplication( "org_" + rand, "app_" + rand );
+ String rand = RandomStringUtils.randomAlphanumeric(5);
+ final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
- final EntityManager em = setup.getEmf().getEntityManager( appId );
+ final EntityManager em = setup.getEmf().getEntityManager(appId);
- final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class );
+ final ReIndexService reIndexService = setup.getInjector().getInstance(ReIndexService.class);
// ----------------- create a bunch of entities
Map<String, Object> entityData = new HashMap<String, Object>() {{
- put( "key1", 1000 );
+ put("key1", 1000);
}};
- final Entity firstEntity = em.create( "thing", entityData );
+ final Entity firstEntity = em.create("thing", entityData);
- final Entity secondEntity = em.create( "thing", entityData);
+ final Entity secondEntity = em.create("thing", entityData);
app.waitForQueueDrainAndRefreshIndex(15000);
// ----------------- test that we can read them, should work fine
- logger.debug( "Read the data" );
+ logger.debug("Read the data");
final String collectionName = "things";
- countEntities( em, collectionName, 2 );
+ countEntities(em, collectionName, 2);
// ----------------- delete the system and application indexes
- logger.debug( "Deleting app index" );
+ logger.debug("Deleting app index");
- deleteIndex( em.getApplicationId() );
+ deleteIndex(em.getApplicationId());
// ----------------- test that we can read them, should fail
@@ -462,14 +458,14 @@ public class RebuildIndexTest extends AbstractCoreIT {
logger.debug("Reading data, should fail this time ");
}
- countEntities( em, collectionName, 0);
+ countEntities(em, collectionName, 0);
// ----------------- rebuild index
final long firstUpdatedTimestamp = firstEntity.getModified();
final long secondUpdatedTimestamp = secondEntity.getModified();
- assertTrue( "second should be updated after second", firstUpdatedTimestamp < secondUpdatedTimestamp );
+ assertTrue("second should be updated after second", firstUpdatedTimestamp < secondUpdatedTimestamp);
try {
@@ -481,36 +477,35 @@ public class RebuildIndexTest extends AbstractCoreIT {
//set our update timestamp
final ReIndexRequestBuilder builder =
- reIndexService.getBuilder().withApplicationId( em.getApplicationId() ).withStartTimestamp(
- updatedTimestamp );
+ reIndexService.getBuilder().withApplicationId(em.getApplicationId()).withStartTimestamp(
+ updatedTimestamp);
- ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex( builder );
+ ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder);
- assertNotNull( status.getJobId(), "JobId is present" );
+ assertNotNull(status.getJobId(), "JobId is present");
- logger.info( "Rebuilt index" );
+ logger.info("Rebuilt index");
- waitForRebuild( status, reIndexService );
+ waitForRebuild(status, reIndexService);
- logger.info( "Rebuilt index" );
+ logger.info("Rebuilt index");
- }
- catch ( Exception ex ) {
- logger.error( "Error rebuilding index", ex );
+ } catch (Exception ex) {
+ logger.error("Error rebuilding index", ex);
fail();
}
// ----------------- test that we can read them
app.waitForQueueDrainAndRefreshIndex(5000);
- countEntities( em, collectionName, 1 );
+ countEntities(em, collectionName, 1);
}
/**
* Wait for the rebuild to occur
*/
- private void waitForRebuild( final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService )
+ private void waitForRebuild(final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService)
throws InterruptedException, IllegalArgumentException {
if (status != null) {
logger.info("waitForRebuild: jobID={}", status.getJobId());
@@ -518,27 +513,26 @@ public class RebuildIndexTest extends AbstractCoreIT {
logger.info("waitForRebuild: error, status = null");
throw new IllegalArgumentException("reindexStatus = null");
}
- while ( true ) {
+ while (true) {
try {
- final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus( status.getJobId() );
+ final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus(status.getJobId());
if (updatedStatus == null) {
logger.info("waitForRebuild: updated status is null");
} else {
logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed());
- if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
+ if (updatedStatus.getStatus() == ReIndexService.Status.COMPLETE) {
break;
}
}
- }
- catch ( IllegalArgumentException iae ) {
+ } catch (IllegalArgumentException iae) {
//swallow. Thrown if our job can't be found. I.E hasn't updated yet
}
- Thread.sleep( 1000 );
+ Thread.sleep(1000);
}
}
@@ -546,22 +540,30 @@ public class RebuildIndexTest extends AbstractCoreIT {
/**
* Delete app index
*/
- private void deleteIndex( UUID appUuid ) {
+ private void deleteIndex(UUID appUuid) {
- Injector injector = SpringResource.getInstance().getBean( Injector.class );
+ Injector injector = SpringResource.getInstance().getBean(Injector.class);
IndexLocationStrategyFactory indexLocationStrategyFactory = injector.getInstance(IndexLocationStrategyFactory.class);
- EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class );
+ EntityIndexFactory eif = injector.getInstance(EntityIndexFactory.class);
- Id appId = new SimpleId( appUuid, Schema.TYPE_APPLICATION );
- ApplicationScope scope = new ApplicationScopeImpl( appId );
+ Id appId = new SimpleId(appUuid, Schema.TYPE_APPLICATION);
+ ApplicationScope scope = new ApplicationScopeImpl(appId);
EntityIndex ei = eif.createEntityIndex(
indexLocationStrategyFactory.getIndexLocationStrategy(scope)
);
- ei.deleteApplication().toBlocking().lastOrDefault( null );
+ ei.deleteApplication().toBlocking().lastOrDefault(null);
app.waitForQueueDrainAndRefreshIndex();
}
+ private int retryReadData(EntityManager em, String collectionName, int expectedEntities, int expectedConnections, int retry) throws Exception {
+ int count = readData(em, collectionName, expectedEntities, expectedConnections);
+ while (count != expectedEntities && --retry >=0) {
+ count = readData(em, collectionName, expectedEntities, expectedConnections);
+ }
+ assertEquals( "Did not get expected entities", expectedEntities, count );
+ return count;
+ }
private int readData( EntityManager em, String collectionName, int expectedEntities, int expectedConnections )
throws Exception {
@@ -598,7 +600,6 @@ public class RebuildIndexTest extends AbstractCoreIT {
}
}
- assertEquals( "Did not get expected entities", expectedEntities, count );
return count;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
index 9c09806..449eac9 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
@@ -28,6 +28,7 @@ import com.google.common.base.Preconditions;
import org.apache.usergrid.corepersistence.index.CollectionDeleteRequestBuilder;
import org.apache.usergrid.corepersistence.index.CollectionDeleteRequestBuilderImpl;
import org.apache.usergrid.corepersistence.index.CollectionDeleteService;
+import org.apache.usergrid.corepersistence.util.CpCollectionUtils;
import org.apache.usergrid.persistence.index.utils.ConversionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,6 +51,7 @@ import com.fasterxml.jackson.jaxrs.json.annotation.JSONP;
import java.util.HashMap;
import java.util.Map;
+import java.util.Properties;
/**
@@ -71,6 +73,34 @@ public class CollectionResource extends ServiceResource {
}
+ /*
+ * Combine the passed settings with the settings already in the collection
+ */
+ private void combineSettings(ServicePayload newSettings, UriInfo ui) throws Exception {
+
+
+ // Get the existing collection settings.
+ ApiResponse response = createApiResponse();
+ response.setAction( "get" );
+ response.setApplication( services.getApplication() );
+ response.setParams( ui.getQueryParameters() );
+
+ executeServiceGetRequestForSettings( ui,response,ServiceAction.GET,null );
+
+ // now add the old settings to the new if they are not alreday in the new.
+ Object oldSettings = response.getData();
+ if (oldSettings instanceof Map) {
+ Map<String,Object> oldProps = (Map) oldSettings;
+ Map<String,Object> newProps = newSettings.getProperties();
+ for (String key : oldProps.keySet()) {
+ if (CpCollectionUtils.getValidSettings().contains(key) && !newProps.containsKey(key)) {
+ Object value = oldProps.get(key);
+ newSettings.setProperty(key, value);
+ }
+ }
+ }
+ }
+
/**
* POST settings for a collection.
*
@@ -87,7 +117,9 @@ public class CollectionResource extends ServiceResource {
@Context UriInfo ui,
@PathParam("itemName") PathSegment itemName,
String body,
- @QueryParam("callback") @DefaultValue("callback") String callback ) throws Exception {
+ @QueryParam("callback") @DefaultValue("callback") String callback,
+ @QueryParam("replace_all") @DefaultValue("true") String replace_all
+ ) throws Exception {
if(logger.isTraceEnabled()){
logger.trace( "CollectionResource.executePostOnSettingsWithCollectionName" );
@@ -110,6 +142,10 @@ public class CollectionResource extends ServiceResource {
ServicePayload payload = getPayload( json );
+ if ("false".equals(replace_all)) {
+ combineSettings(payload, ui);
+ }
+
executeServicePostRequestForSettings( ui,response, ServiceAction.POST, payload );
return response;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index b8baa3d..eff9f2a 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -58,10 +58,10 @@ public class CollectionsResourceIT extends AbstractRestIT {
private static final Logger log = LoggerFactory.getLogger( CollectionsResourceIT.class );
@Test
- public void testValidSettings() throws IOException {
+ public void testPostValidSettings() throws IOException {
String randomizer = RandomStringUtils.randomAlphanumeric(10);
- String collectionName = "collection" + randomizer;
+ String collectionName = "collection_" + randomizer;
//Create test collection with test entity that is full text indexed.
Entity testEntity = new Entity();
@@ -91,12 +91,15 @@ public class CollectionsResourceIT extends AbstractRestIT {
List list = (List) feildsValue;
assertEquals(1, list.size() );
assertEquals( "one", list.get(0) );
-
}
+ /**
+ * test that invalid settings are ignored
+ * @throws IOException
+ */
@Test
- public void testInvalidSettings() throws IOException {
+ public void testPostInvalidSettings() throws IOException {
String randomizer = RandomStringUtils.randomAlphanumeric(10);
String collectionName = "collection" + randomizer;
@@ -126,6 +129,78 @@ public class CollectionsResourceIT extends AbstractRestIT {
}
+
+ /**
+ * test that we can add new properties and modify existing ones.
+ * @throws IOException
+ */
+ @Test
+ public void testAddNewSettings() throws IOException {
+
+ String randomizer = RandomStringUtils.randomAlphanumeric(10);
+ String collectionName = "collection_" + randomizer;
+
+ //Create test collection with test entity that is full text indexed.
+ Entity testEntity = new Entity();
+
+ testEntity.put( "one","value" );
+ testEntity.put( "two","value" );
+ app().collection( collectionName ).post( testEntity );
+
+ //field "fields" is required.
+ Entity payload = new Entity();
+ payload.put( "fields", Collections.singletonList("one"));
+
+ //Post index to the collection metadata
+ app().collection( collectionName ).collection( "_settings" ).post( payload );
+ waitForQueueDrainAndRefreshIndex();
+
+ // Now add a new setting 'queueIndex'
+
+ payload = new Entity();
+ payload.put( "queueIndex", "direct");
+
+ Map<String, String> queryParams = Collections.singletonMap("replace_all","false");
+ Entity settings = app().collection( collectionName ).collection( "_settings" ).post( payload , queryParams );
+ waitForQueueDrainAndRefreshIndex();
+
+ // assert that both old and new are in the setting.
+
+ Object queueIndexValue = settings.get("queueIndex");
+ assertEquals( "direct", queueIndexValue);
+
+ Object feildsValue = settings.get("fields");
+ assertNotNull(feildsValue);
+ assertTrue(feildsValue instanceof List);
+ List list = (List) feildsValue;
+ assertEquals(1, list.size() );
+ assertEquals( "one", list.get(0) );
+
+ // now replace the settings
+
+ payload = new Entity();
+ payload.put( "queueIndex", "async");
+ payload.put( "fields", Collections.singletonList("two"));
+
+ settings = app().collection( collectionName ).collection( "_settings" ).post( payload , queryParams );
+ waitForQueueDrainAndRefreshIndex();
+
+ // assert that they have new values
+
+ queueIndexValue = settings.get("queueIndex");
+ assertEquals( "async", queueIndexValue);
+
+ feildsValue = settings.get("fields");
+ assertNotNull(feildsValue);
+ assertTrue(feildsValue instanceof List);
+ list = (List) feildsValue;
+ assertEquals(1, list.size() );
+ assertEquals( "two", list.get(0) );
+
+ }
+
+
+
@Test
public void testStaleEntries() throws IOException {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
index af87ca5..66276bd 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
@@ -75,9 +75,21 @@ public class UserResourceIT extends AbstractRestIT {
usersResource = this.app().collection("users");
userResource = this.app().collection("user");
+ // make the queueindex direct to speed up the tests.
+ Entity payload = new Entity();
+ payload.put( "queueIndex", "direct");
+
+ //Post index to the collection metadata
+ app().collection( "user" ).collection( "_settings" ).post( payload );
+ app().collection( "users" ).collection( "_settings" ).post( payload );
+
waitForQueueDrainAndRefreshIndex();
}
+ public void waitForQueueDrainAndRefreshIndex() {
+ waitForQueueDrainAndRefreshIndex(50);
+ }
+
@Test
public void usernameQuery() throws IOException {
String ql = "username = 'user*'";
@@ -398,7 +410,7 @@ public class UserResourceIT extends AbstractRestIT {
@Test
- public void clientNameQuery() {
+ public void clientNameQuery() throws Exception {
String username = "username";
@@ -410,7 +422,18 @@ public class UserResourceIT extends AbstractRestIT {
UUID createdId = entity.getUuid();
waitForQueueDrainAndRefreshIndex();
- Collection results = usersResource.get(new QueryParameters().setQuery(String.format("name = '%s'", name)));
+
+ QueryParameters queryParameters = new QueryParameters().setQuery(String.format("name = '%s'", name));
+ Collection results = usersResource.get(queryParameters);
+
+ int retry = 30;
+ while (results.getResponse().getEntities().size() == 0 && --retry >= 0) {
+ Thread.sleep(1000);
+ results = usersResource.get(queryParameters);
+ logger.info("clientNameQuery results size {} ", results.getResponse().getEntities().size());
+ }
+ assertEquals(1 , results.getResponse().getEntities().size());
+
entity = new User(results.getResponse().getEntities().get(0));
assertEquals(createdId, entity.getUuid());
}
@@ -1214,7 +1237,11 @@ public class UserResourceIT extends AbstractRestIT {
ql = "uuid = " + userId;
Collection response = this.app().collection("curts").get(new QueryParameters().setQuery(ql));
-
+ int retry = 30;
+ while (response.getResponse().getEntities().isEmpty() &&--retry >= 0) {
+ Thread.sleep(1000);
+ }
+ assertFalse(response.getResponse().getEntities().isEmpty());
assertEquals(response.getResponse().getEntities().get(0).get("uuid").toString(), userId.toString());
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java
index d1c7ab4..9b1ad82 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource/endpoints/CollectionEndpoint.java
@@ -18,9 +18,7 @@ package org.apache.usergrid.rest.test.resource.endpoints;
import java.io.IOException;
import java.io.StringReader;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;
@@ -229,7 +227,7 @@ public class CollectionEndpoint extends NamedResource {
}
/**
- * Post an entity to a collection.
+ * Post an entity to a collection with query params
*
* <pre>
* app.collection("users").post(entity);
@@ -237,15 +235,20 @@ public class CollectionEndpoint extends NamedResource {
* </pre>
*/
public org.apache.usergrid.rest.test.resource.model.Entity post(
- org.apache.usergrid.rest.test.resource.model.Entity payload){
+ org.apache.usergrid.rest.test.resource.model.Entity payload, Map<String,String> queryParams){
String acceptHeader = MediaType.APPLICATION_JSON;
if (this.acceptHeaders.size() > 0) {
acceptHeader = StringUtils.join(this.acceptHeaders, ',');
}
+ WebTarget target = getTarget(true);
+ for (String key : queryParams.keySet()) {
+ target = target.queryParam(key, queryParams.get(key));
+ }
+
// use string type so we can log actual response from server
- String responseString = getTarget( true )
+ String responseString = target
.request()
.accept(acceptHeader)
.post( javax.ws.rs.client.Entity.json( payload ), String.class);
@@ -266,6 +269,21 @@ public class CollectionEndpoint extends NamedResource {
return new org.apache.usergrid.rest.test.resource.model.Entity(response);
}
+
+ /**
+ * Post an entity to a collection.
+ *
+ * <pre>
+ * app.collection("users").post(entity);
+ * POST /users {"color","red"}
+ * </pre>
+ */
+ public org.apache.usergrid.rest.test.resource.model.Entity post(
+ org.apache.usergrid.rest.test.resource.model.Entity payload){
+
+ return post(payload, Collections.emptyMap());
+ }
+
public org.apache.usergrid.rest.test.resource.model.Entity post() {
String acceptHeader = MediaType.APPLICATION_JSON;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
index 9b3dfcd..bab3812 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
@@ -49,15 +49,17 @@ public abstract class AbstractServiceNotificationIT extends AbstractServiceIT {
protected Notification notificationWaitForComplete(Notification notification)
throws Exception {
- long timeout = System.currentTimeMillis() + 120000;
- while (System.currentTimeMillis() < timeout) {
- app.waitForQueueDrainAndRefreshIndex(1000);
+
+ int retry = 18; // 3 mins 18 * 10 seconds
+ while (-- retry > 0) {
+ logger.info("notificationWaitForComplete {} retry {}", notification.getUuid(), retry);
+ app.waitForQueueDrainAndRefreshIndex(10000);
notification = app.getEntityManager().get(notification.getUuid(), Notification.class);
if (notification.getFinished() != null) {
return notification;
}
}
- fail("Notification failed to complete");
+ fail("Notification failed to complete error message " + notification.getErrorMessage());
return null;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
index 4fb9add..09ecc59 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/apns/NotificationsServiceIT.java
@@ -323,13 +323,15 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
// delay until the scheduler has time to run
- logger.info("Sleeping while the scheduler does its work");
- Thread.sleep(5000);
-
-
- notification = app.getEntityManager().get(e.getUuid(), Notification.class);
-
- assertEquals(Notification.State.FINISHED, notification.getState());
+ Notification.State notificationState = Notification.State.SCHEDULED;
+ int retry = 60;
+ do {
+ logger.info("Sleeping while the scheduler does its work " + retry);
+ Thread.sleep(1000);
+ notificationState = app.getEntityManager().get(e.getUuid(), Notification.class).getState();
+ } while (Notification.State.FINISHED != notificationState && --retry >= 0 );
+
+ assertEquals(Notification.State.FINISHED, notificationState);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2110bfbd/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index ecc5443..d403d5b 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -632,10 +632,17 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
// receipts are created and queried, wait a bit longer for this to happen as indexing
- app.waitForQueueDrainAndRefreshIndex(5000);
+ app.waitForQueueDrainAndRefreshIndex(2000);
// get the receipts entity IDs
List<EntityRef> receipts = getNotificationReceipts(notification);
+
+ int retry = 30;
+ while (receipts.size() == 0 && --retry >=0 ) {
+ app.waitForQueueDrainAndRefreshIndex(1000);
+ receipts = getNotificationReceipts(notification);
+ }
+
assertEquals(1, receipts.size());
// Validate the error is the correct type InvalidRegistration
[2/3] usergrid git commit: Fix an error in UserResourceIT unit test
Posted by md...@apache.org.
Fix an error in UserResourceIT unit test
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c172133e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c172133e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c172133e
Branch: refs/heads/3.0.0_experimental
Commit: c172133e3d89e27fc099eb6c01ba3be8b3647654
Parents: 2110bfb
Author: Peter Johnson <pj...@apigee.com>
Authored: Thu Nov 16 21:09:14 2017 -0800
Committer: Peter Johnson <pj...@apigee.com>
Committed: Thu Nov 16 21:09:14 2017 -0800
----------------------------------------------------------------------
.../applications/collection/users/UserResourceIT.java | 12 ------------
1 file changed, 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c172133e/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
index 66276bd..1fd6b13 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/users/UserResourceIT.java
@@ -75,21 +75,9 @@ public class UserResourceIT extends AbstractRestIT {
usersResource = this.app().collection("users");
userResource = this.app().collection("user");
- // make the queueindex direct to speed up the tests.
- Entity payload = new Entity();
- payload.put( "queueIndex", "direct");
-
- //Post index to the collection metadata
- app().collection( "user" ).collection( "_settings" ).post( payload );
- app().collection( "users" ).collection( "_settings" ).post( payload );
-
waitForQueueDrainAndRefreshIndex();
}
- public void waitForQueueDrainAndRefreshIndex() {
- waitForQueueDrainAndRefreshIndex(50);
- }
-
@Test
public void usernameQuery() throws IOException {
String ql = "username = 'user*'";
[3/3] usergrid git commit: Upgrade Elastic search t0 5.6.3
Posted by md...@apache.org.
Upgrade Elastic search t0 5.6.3
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5465d807
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5465d807
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5465d807
Branch: refs/heads/3.0.0_experimental
Commit: 5465d807ef3cbe8521b623b9d520be808745daa3
Parents: c172133
Author: Peter Johnson <pj...@apigee.com>
Authored: Tue Oct 24 16:07:37 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Nov 20 09:51:13 2017 -0800
----------------------------------------------------------------------
stack/corepersistence/pom.xml | 2 +-
stack/corepersistence/queryindex/pom.xml | 6 ++
.../index/impl/EsEntityIndexImpl.java | 66 ++++++------
.../index/impl/EsIndexProducerImpl.java | 23 ++++-
.../persistence/index/impl/EsProvider.java | 18 ++--
.../persistence/index/impl/EsQueryVistor.java | 100 ++++++++-----------
.../index/impl/NoOpFilterBuilder.java | 13 +--
.../index/impl/NoOpQueryBuilder.java | 33 +++++-
.../impl/SearchRequestBuilderStrategy.java | 36 ++++---
.../persistence/index/impl/SortBuilder.java | 8 +-
.../index/query/tree/QueryVisitor.java | 10 +-
.../persistence/index/usergrid-mappings.json | 2 -
12 files changed, 174 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index 45aad4b..ff09cf2 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -82,7 +82,7 @@ limitations under the License.
<commons.io.version>2.4</commons.io.version>
<commons.lang.version>3.1</commons.lang.version>
<datastax.version>2.1.10.3</datastax.version>
- <elasticsearch.version>1.7.5</elasticsearch.version>
+ <elasticsearch.version>5.6.3</elasticsearch.version>
<fasterxml-uuid.version>3.1.3</fasterxml-uuid.version>
<guava.version>18.0</guava.version>
<guice.version>4.0-beta5</guice.version>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml
index 11856cb..c5401e4 100644
--- a/stack/corepersistence/queryindex/pom.xml
+++ b/stack/corepersistence/queryindex/pom.xml
@@ -101,6 +101,12 @@
<type>jar</type>
</dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>transport</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.elasticsearch</groupId>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 211cf70..9a0af48 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -47,10 +47,8 @@ import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ListenableActionFuture;
-import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.action.*;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
@@ -59,22 +57,21 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
-import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
+import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
-import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.DeleteByQueryAction;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
-import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
+import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -206,13 +203,13 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
//Create index
try {
final AdminClient admin = esProvider.getClient().admin();
- Settings settings = ImmutableSettings.settingsBuilder()
+ Settings settings = Settings.builder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", numberOfReplicas)
//dont' allow unmapped queries, and don't allow dynamic mapping
.put("index.query.parse.allow_unmapped_fields", false)
.put("index.mapper.dynamic", false)
- .put("action.write_consistency", writeConsistency)
+ // .put("action.write_consistency", writeConsistency)
.build();
//Added For Graphite Metrics
@@ -230,7 +227,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
- } catch (IndexAlreadyExistsException e) {
+ } catch (ResourceAlreadyExistsException e) {
logger.info("Index Name [{}] already exists", indexName);
}
/**
@@ -244,7 +241,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
testNewIndex();
- } catch (IndexAlreadyExistsException expected) {
+ } catch (ResourceAlreadyExistsException expected) {
// this is expected to happen if index already exists, it's a no-op and swallow
} catch (IOException e) {
throw new RuntimeException("Unable to initialize index", e);
@@ -452,8 +449,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
for (SortPredicate sortPredicate : parsedQuery.getSortPredicates() ){
hasGeoSortPredicates = visitor.getGeoSorts().contains(sortPredicate.getPropertyName());
}
-
-
+
final String cacheKey = applicationScope.getApplication().getUuid().toString()+"_"+searchEdge.getEdgeName();
final Object totalEdgeSizeFromCache = sizeCache.getIfPresent(cacheKey);
long totalEdgeSizeInBytes;
@@ -498,6 +494,9 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
final Timer.Context timerContext = searchTimer.time();
try {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Query to execute = {}", srb.toString());
+ }
searchResponse = srb.execute().actionGet();
}
@@ -594,21 +593,25 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
//Added For Graphite Metrics
return Observable.from( indexes ).flatMap( index -> {
- final ListenableActionFuture<DeleteByQueryResponse> response =
- esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
+ ListenableActionFuture<BulkByScrollResponse> response =
+ DeleteByQueryAction.INSTANCE.newRequestBuilder( esProvider.getClient())
+ .filter(tqb)
+ .source(indexes)
+ .execute();
+
- response.addListener( new ActionListener<DeleteByQueryResponse>() {
+ response.addListener( new ActionListener<BulkByScrollResponse>() {
@Override
- public void onResponse( DeleteByQueryResponse response ) {
+ public void onResponse( BulkByScrollResponse response ) {
checkDeleteByQueryResponse( tqb, response );
}
-
@Override
- public void onFailure( Throwable e ) {
+ public void onFailure(Exception e) {
logger.error( "Failed on delete index", e.getMessage() );
}
+
} );
return Observable.from( response );
} ).doOnError( t -> logger.error( "Failed on delete application", t.getMessage() ) );
@@ -618,17 +621,14 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
/**
* Validate the response doesn't contain errors, if it does, fail fast at the first error we encounter
*/
- private void checkDeleteByQueryResponse( final QueryBuilder query, final DeleteByQueryResponse response ) {
+ private void checkDeleteByQueryResponse( final QueryBuilder query, final BulkByScrollResponse response ) {
- for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) {
- final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures();
+ List<BulkItemResponse.Failure> failures = response.getBulkFailures();
- for ( ShardOperationFailedException failedException : failures ) {
- logger.error("Unable to delete by query {}. Failed with code {} and reason {} on shard {} in index {}",
- query.toString(),
- failedException.status().getStatus(), failedException.reason(),
- failedException.shardId(), failedException.index() );
- }
+
+ for ( BulkItemResponse.Failure failure : failures ) {
+ logger.error("Unable to delete by query {}. Failed with code {} and reason {} in index {}",
+ query.toString(), failure.getStatus() , failure.getMessage(), failure.getIndex());
}
}
@@ -818,7 +818,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
.actionGet();
final CommonStats indexStats = statsResponse.getIndex(indexName).getTotal();
indexSize = indexStats.getStore().getSizeInBytes();
- } catch (IndexMissingException e) {
+ } catch (IndexNotFoundException e) {
// if for some reason the index size does not exist,
// log an error and we can assume size is 0 as it doesn't exist
logger.error("Unable to get size for index {} due to IndexMissingException for app {}",
@@ -837,7 +837,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
private long getEntitySizeAggregation( final SearchRequestBuilder builder ) {
final String key = "entitySize";
- SumBuilder sumBuilder = new SumBuilder(key);
+ SumAggregationBuilder sumBuilder = new SumAggregationBuilder(key);
sumBuilder.field("entitySize");
builder.addAggregation(sumBuilder);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index 8f58ef7..468a99e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -26,10 +26,10 @@ import com.codahale.metrics.Histogram;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.elasticsearch.action.WriteConsistencyLevel;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.client.Client;
import org.elasticsearch.rest.RestStatus;
import org.slf4j.Logger;
@@ -169,9 +169,26 @@ public class EsIndexProducerImpl implements IndexProducer {
* initialize request
*/
private BulkRequestBuilder initRequest() {
+
BulkRequestBuilder bulkRequest = client.prepareBulk();
- bulkRequest.setConsistencyLevel( WriteConsistencyLevel.fromString( config.getWriteConsistencyLevel() ) );
- bulkRequest.setRefresh( config.isForcedRefresh() );
+
+ String refreshPolicyConfig = String.valueOf(config.isForcedRefresh());
+ bulkRequest.setRefreshPolicy(refreshPolicyConfig);
+
+ String consistencyLevel = config.getWriteConsistencyLevel();
+
+ if ("one".equals(consistencyLevel)) {
+ bulkRequest.setWaitForActiveShards(1);
+ }
+
+ if ("all".equals(consistencyLevel)) {
+ bulkRequest.setWaitForActiveShards(ActiveShardCount.ALL);
+ }
+
+ if ("none".equals(consistencyLevel)) {
+ bulkRequest.setWaitForActiveShards(ActiveShardCount.NONE);
+ }
+
return bulkRequest;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
index e81aa0f..fe35a02 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
@@ -24,11 +24,10 @@ import java.net.UnknownHostException;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -123,7 +122,8 @@ public class EsProvider {
final String clusterName = indexFig.getClusterName();
final int port = indexFig.getPort();
- ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put( "cluster.name", clusterName )
+ Settings.Builder settings = Settings.builder()
+ .put( "cluster.name", clusterName )
.put( "client.transport.sniff", true );
String nodeName = indexFig.getNodeName();
@@ -142,11 +142,15 @@ public class EsProvider {
settings.put( "node.name", nodeName );
- TransportClient transportClient = new TransportClient( settings.build() );
+ TransportClient transportClient = new PreBuiltTransportClient(settings.build());
// we will connect to ES on all configured hosts
for ( String host : indexFig.getHosts().split( "," ) ) {
- transportClient.addTransportAddress( new InetSocketTransportAddress( host, port ) );
+ try {
+ transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
+ } catch (UnknownHostException uhe) {
+ logger.error( "Couldn't resolve hostname {} to use as ES node ", host );
+ }
}
return transportClient;
@@ -181,7 +185,7 @@ public class EsProvider {
final String hostString = hosts.toString();
- Settings settings = ImmutableSettings.settingsBuilder()
+ Settings settings = Settings.builder()
.put( "cluster.name", clusterName )
@@ -200,7 +204,7 @@ public class EsProvider {
logger.trace("Creating ElasticSearch client with settings: {}", settings.getAsMap());
}
- Node node = NodeBuilder.nodeBuilder().settings( settings ).client( true ).data( false ).node();
+ Node node = new Node( settings );
return node.client();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 4dd0d24..7adb332 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -22,20 +22,11 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.Stack;
import java.util.UUID;
+import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.query.BoolFilterBuilder;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.MatchQueryBuilder;
-import org.elasticsearch.index.query.NestedFilterBuilder;
-import org.elasticsearch.index.query.NestedQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.RangeFilterBuilder;
-import org.elasticsearch.index.query.TermFilterBuilder;
-import org.elasticsearch.index.query.WildcardQueryBuilder;
+import org.elasticsearch.index.query.*;
+
import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.slf4j.Logger;
@@ -75,7 +66,7 @@ public class EsQueryVistor implements QueryVisitor {
/**
* Our queryBuilders for filter operations
*/
- private final Stack<FilterBuilder> filterBuilders = new Stack<>();
+ private final Stack<QueryBuilder> filterBuilders = new Stack<>();
private final GeoSortFields geoSortFields = new GeoSortFields();
@@ -89,12 +80,12 @@ public class EsQueryVistor implements QueryVisitor {
//get all the right
final QueryBuilder rightQuery = queryBuilders.pop();
- final FilterBuilder rightFilter = filterBuilders.pop();
+ final QueryBuilder rightFilter = filterBuilders.pop();
//get all the left
final QueryBuilder leftQuery = queryBuilders.pop();
- final FilterBuilder leftFilter = filterBuilders.pop();
+ final QueryBuilder leftFilter = filterBuilders.pop();
//push our boolean filters
@@ -131,7 +122,7 @@ public class EsQueryVistor implements QueryVisitor {
//use left and right
if ( useLeftFilter && useRightFilter ) {
- final BoolFilterBuilder fb = FilterBuilders.boolFilter().must(leftFilter).must(rightFilter);
+ final BoolQueryBuilder fb = QueryBuilders.boolQuery().must(leftFilter).must(rightFilter);
filterBuilders.push( fb );
}
@@ -157,12 +148,12 @@ public class EsQueryVistor implements QueryVisitor {
op.getRight().visit( this );
final QueryBuilder rightQuery = queryBuilders.pop();
- final FilterBuilder rightFilter = filterBuilders.pop();
+ final QueryBuilder rightFilter = filterBuilders.pop();
//get all the left
final QueryBuilder leftQuery = queryBuilders.pop();
- final FilterBuilder leftFilter = filterBuilders.pop();
+ final QueryBuilder leftFilter = filterBuilders.pop();
final boolean useLeftQuery = use( leftQuery );
@@ -193,7 +184,7 @@ public class EsQueryVistor implements QueryVisitor {
//use left and right
if ( useLeftFilter && useRightFilter ) {
- final BoolFilterBuilder fb = FilterBuilders.boolFilter().should( leftFilter ).should( rightFilter );
+ final BoolQueryBuilder fb = QueryBuilders.boolQuery().should( leftFilter ).should( rightFilter );
filterBuilders.push( fb );
}
@@ -232,12 +223,12 @@ public class EsQueryVistor implements QueryVisitor {
queryBuilders.push( NoOpQueryBuilder.INSTANCE );
}
- final FilterBuilder notFilterBuilder = filterBuilders.pop();
+ final QueryBuilder notFilterBuilder = filterBuilders.pop();
//push the filter in
if ( use( notFilterBuilder ) ) {
- final FilterBuilder notFilter = FilterBuilders.boolFilter().mustNot( notFilterBuilder ) ;
+ final QueryBuilder notFilter = QueryBuilders.boolQuery().mustNot( notFilterBuilder ) ;
//just the root node
if(!rootNode) {
@@ -245,7 +236,7 @@ public class EsQueryVistor implements QueryVisitor {
}
//not the root node, we have to select all to subtract from with the NOT statement
else{
- final FilterBuilder selectAllFilter = FilterBuilders.boolFilter().must( FilterBuilders.matchAllFilter()) .must( notFilter );
+ final QueryBuilder selectAllFilter = QueryBuilders.boolQuery().must( QueryBuilders.matchAllQuery()) .must( notFilter );
filterBuilders.push( selectAllFilter );
}
@@ -293,9 +284,10 @@ public class EsQueryVistor implements QueryVisitor {
float distance = op.getDistance().getFloatValue();
- final FilterBuilder fb =
- FilterBuilders.geoDistanceFilter( IndexingUtils.FIELD_LOCATION_NESTED ).lat( lat ).lon( lon )
- .distance( distance, DistanceUnit.METERS );
+ final QueryBuilder fb =
+ QueryBuilders.geoDistanceQuery( IndexingUtils.FIELD_LOCATION_NESTED )
+ .point(lat, lon)
+ .distance( distance, DistanceUnit.METERS );
filterBuilders.push( fieldNameTerm( name, fb ) );
@@ -306,10 +298,11 @@ public class EsQueryVistor implements QueryVisitor {
//this geoSort won't has a sort on it
final GeoDistanceSortBuilder geoSort =
- SortBuilders.geoDistanceSort( IndexingUtils.FIELD_LOCATION_NESTED ).unit( DistanceUnit.METERS )
- .geoDistance(GeoDistance.SLOPPY_ARC).point(lat, lon);
+ SortBuilders.geoDistanceSort( IndexingUtils.FIELD_LOCATION_NESTED, lat, lon )
+ .unit( DistanceUnit.METERS )
+ .geoDistance(GeoDistance.ARC).point(lat, lon);
- final TermFilterBuilder sortPropertyName = sortPropertyTermFilter(name);
+ final TermQueryBuilder sortPropertyName = sortPropertyTermFilter(name);
geoSort.setNestedFilter( sortPropertyName );
@@ -328,8 +321,8 @@ public class EsQueryVistor implements QueryVisitor {
final Object value = op.getLiteral().getValue();
- final RangeFilterBuilder termQuery =
- FilterBuilders.rangeFilter( getFieldNameForType( value ) ).lt(sanitize(value));
+ final RangeQueryBuilder termQuery =
+ QueryBuilders.rangeQuery( getFieldNameForType( value ) ).lt(sanitize(value));
queryBuilders.push( NoOpQueryBuilder.INSTANCE );
@@ -346,8 +339,8 @@ public class EsQueryVistor implements QueryVisitor {
final Object value = op.getLiteral().getValue();
- final RangeFilterBuilder termQuery =
- FilterBuilders.rangeFilter( getFieldNameForType( value ) ).lte(sanitize(value));
+ final RangeQueryBuilder termQuery =
+ QueryBuilders.rangeQuery( getFieldNameForType( value ) ).lte(sanitize(value));
queryBuilders.push( NoOpQueryBuilder.INSTANCE );
@@ -383,8 +376,8 @@ public class EsQueryVistor implements QueryVisitor {
}
//it's an exact match, use a filter
- final TermFilterBuilder termFilter =
- FilterBuilders.termFilter( IndexingUtils.FIELD_STRING_NESTED_UNANALYZED, stringValue );
+ final TermQueryBuilder termFilter =
+ QueryBuilders.termQuery( IndexingUtils.FIELD_STRING_NESTED_UNANALYZED, stringValue );
queryBuilders.push( NoOpQueryBuilder.INSTANCE );
filterBuilders.push( fieldNameTerm( name, termFilter ) );
@@ -394,8 +387,8 @@ public class EsQueryVistor implements QueryVisitor {
// assume all other types need prefix
- final TermFilterBuilder termQuery =
- FilterBuilders.termFilter(getFieldNameForType(value), sanitize(value));
+ final TermQueryBuilder termQuery =
+ QueryBuilders.termQuery(getFieldNameForType(value), sanitize(value));
filterBuilders.push( fieldNameTerm( name, termQuery ) );
@@ -409,8 +402,8 @@ public class EsQueryVistor implements QueryVisitor {
final Object value = op.getLiteral().getValue();
- final RangeFilterBuilder rangeQuery =
- FilterBuilders.rangeFilter( getFieldNameForType( value ) ).gt(sanitize(value));
+ final RangeQueryBuilder rangeQuery =
+ QueryBuilders.rangeQuery( getFieldNameForType( value ) ).gt(sanitize(value));
filterBuilders.push( fieldNameTerm( name, rangeQuery ) );
@@ -424,8 +417,8 @@ public class EsQueryVistor implements QueryVisitor {
Object value = op.getLiteral().getValue();
- final RangeFilterBuilder rangeQuery =
- FilterBuilders.rangeFilter( getFieldNameForType( value ) ).gte(sanitize(value));
+ final RangeQueryBuilder rangeQuery =
+ QueryBuilders.rangeQuery( getFieldNameForType( value ) ).gte(sanitize(value));
filterBuilders.push(fieldNameTerm(name, rangeQuery));
@@ -434,12 +427,12 @@ public class EsQueryVistor implements QueryVisitor {
@Override
- public Optional<FilterBuilder> getFilterBuilder() {
+ public Optional<QueryBuilder> getFilterBuilder() {
if ( filterBuilders.empty() ) {
return Optional.absent();
}
- final FilterBuilder builder = filterBuilders.peek();
+ final QueryBuilder builder = filterBuilders.peek();
if ( !use( builder ) ) {
return Optional.absent();
@@ -484,26 +477,26 @@ public class EsQueryVistor implements QueryVisitor {
booleanQuery.must( fieldValueQuery );
- return QueryBuilders.nestedQuery(IndexingUtils.ENTITY_FIELDS, booleanQuery);
+ return QueryBuilders.nestedQuery(IndexingUtils.ENTITY_FIELDS, booleanQuery, ScoreMode.None);
}
/**
* Generate the field name term for the field name for filters
- */
- private NestedFilterBuilder fieldNameTerm( final String fieldName, final FilterBuilder fieldValueBuilder ) {
+ *
+ private NestedQueryBuilder fieldNameTerm( final String fieldName, final QueryBuilder fieldValueBuilder ) {
- final BoolFilterBuilder booleanQuery = FilterBuilders.boolFilter();
+ final BoolQueryBuilder booleanQuery = QueryBuilders.boolQuery();
- booleanQuery.must( FilterBuilders.termFilter( IndexingUtils.FIELD_NAME_NESTED, fieldName ) );
+ booleanQuery.must( QueryBuilders.termQuery( IndexingUtils.FIELD_NAME_NESTED, fieldName ) );
booleanQuery.must( fieldValueBuilder );
- return FilterBuilders.nestedFilter( IndexingUtils.ENTITY_FIELDS, booleanQuery );
+ return QueryBuilders.nestedQuery( IndexingUtils.ENTITY_FIELDS, booleanQuery );
}
-
+*/
/**
* Get the field name for the primitive type
*/
@@ -578,14 +571,7 @@ public class EsQueryVistor implements QueryVisitor {
* Return false if our element is a no-op, true otherwise
*/
private boolean use( final QueryBuilder queryBuilder ) {
- return queryBuilder != NoOpQueryBuilder.INSTANCE;
+ return queryBuilder != NoOpQueryBuilder.INSTANCE && queryBuilder != NoOpFilterBuilder.INSTANCE;
}
-
- /**
- * Return false if our element is a no-op, true otherwise
- */
- private boolean use( final FilterBuilder filterBuilder ) {
- return filterBuilder != NoOpFilterBuilder.INSTANCE;
- }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java
index 9eca433..780ecf3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java
@@ -20,22 +20,11 @@
package org.apache.usergrid.persistence.index.impl;
-import java.io.IOException;
-
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.BaseFilterBuilder;
-
-
/**
* Marker interface that does nothing
*/
-public class NoOpFilterBuilder extends BaseFilterBuilder {
+public class NoOpFilterBuilder extends NoOpQueryBuilder {
public static final NoOpFilterBuilder INSTANCE = new NoOpFilterBuilder();
-
- @Override
- protected void doXContent( final XContentBuilder builder, final Params params ) throws IOException {
- //no op
- }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java
index 2498812..69b39ee 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java
@@ -22,20 +22,49 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.IOException;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.BaseQueryBuilder;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
/**
* Marker interface that does nothing
*/
-public class NoOpQueryBuilder extends BaseQueryBuilder implements QueryBuilder {
+public class NoOpQueryBuilder extends AbstractQueryBuilder implements QueryBuilder {
public static final NoOpQueryBuilder INSTANCE = new NoOpQueryBuilder();
@Override
+ protected void doWriteTo(StreamOutput out) throws IOException {
+
+ }
+
+ @Override
protected void doXContent( final XContentBuilder builder, final Params params ) throws IOException {
//no op
}
+
+ @Override
+ protected Query doToQuery(QueryShardContext context) throws IOException {
+ return null;
+ }
+
+ @Override
+ protected boolean doEquals(AbstractQueryBuilder other) {
+ return false;
+ }
+
+ @Override
+ protected int doHashCode() {
+ return 0;
+ }
+
+
+ @Override
+ public String getWriteableName() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 5812c6f..3119b4a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -23,11 +23,10 @@ package org.apache.usergrid.persistence.index.impl;
import org.apache.usergrid.persistence.index.IndexAlias;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.index.query.BoolFilterBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.TermFilterBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
@@ -50,7 +49,7 @@ import com.google.common.base.Preconditions;
import java.util.List;
import java.util.Map;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
import static org.apache.usergrid.persistence.index.impl.SortBuilder.sortPropertyTermFilter;
@@ -145,6 +144,7 @@ public class SearchRequestBuilderStrategy {
//sort by the entity id if our times are equal
//srb.addSort( SortBuilders.fieldSort( IndexingUtils.ENTITY_ID_FIELDNAME ).order( SortOrder.ASC ) );
+
return;
}
@@ -205,7 +205,7 @@ public class SearchRequestBuilderStrategy {
* Create our filter builder. We need to restrict our results on edge search, as well as on types, and any filters
* that came from the grammar.
*/
- private FilterBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor visitor,
+ private QueryBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor visitor,
final SearchTypes searchTypes ) {
String context = createContextName( applicationScope, searchEdge );
@@ -218,10 +218,10 @@ public class SearchRequestBuilderStrategy {
// Do we need to put the context term first for performance?
//make sure we have entity in the context
- BoolFilterBuilder boolQueryFilter = FilterBuilders.boolFilter();
+ BoolQueryBuilder boolQueryFilter = QueryBuilders.boolQuery();
//add our edge search
- boolQueryFilter.must( FilterBuilders.termFilter( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) );
+ boolQueryFilter.must( QueryBuilders.termQuery( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) );
/**
@@ -231,19 +231,23 @@ public class SearchRequestBuilderStrategy {
if ( sourceTypes.length > 0 ) {
- final FilterBuilder[] typeTerms = new FilterBuilder[sourceTypes.length];
+ final QueryBuilder[] typeTerms = new QueryBuilder[sourceTypes.length];
+
+ BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
for ( int i = 0; i < sourceTypes.length; i++ ) {
- typeTerms[i] = FilterBuilders.termFilter( IndexingUtils.ENTITY_TYPE_FIELDNAME, sourceTypes[i] );
+ typeTerms[i] = QueryBuilders.termQuery( IndexingUtils.ENTITY_TYPE_FIELDNAME, sourceTypes[i] );
+ boolQuery.filter(typeTerms[i]);
}
//add all our types, 1 type must match per query
- boolQueryFilter.must( FilterBuilders.orFilter( typeTerms ) );
+ //boolQueryFilter.must( QueryBuilders.orFilter( typeTerms ) );
+ boolQueryFilter.must( boolQuery );
}
//if we have a filter from our visitor, add it
- Optional<FilterBuilder> queryBuilder = visitor.getFilterBuilder();
+ Optional<QueryBuilder> queryBuilder = visitor.getFilterBuilder();
if ( queryBuilder.isPresent() ) {
boolQueryFilter.must( queryBuilder.get() );
@@ -263,9 +267,13 @@ public class SearchRequestBuilderStrategy {
private FieldSortBuilder createSort( final SortOrder sortOrder, final String fieldName,
final String propertyName ) {
- final TermFilterBuilder propertyFilter = sortPropertyTermFilter( propertyName );
+ final TermQueryBuilder propertyFilter = QueryBuilders.termQuery( ENTITY_FIELDS + "." + IndexingUtils.FIELD_NAME, propertyName );
+ FieldSortBuilder fieldSortBuilder = SortBuilders.fieldSort( fieldName )
+ .order( sortOrder )
+ .setNestedFilter( propertyFilter )
+ .setNestedPath(ENTITY_FIELDS );
- return SortBuilders.fieldSort( fieldName ).order( sortOrder ).setNestedFilter( propertyFilter );
+ return fieldSortBuilder;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java
index ad78be7..f487747 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java
@@ -24,8 +24,8 @@
package org.apache.usergrid.persistence.index.impl;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.TermFilterBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermQueryBuilder;
/**
@@ -37,8 +37,8 @@ public class SortBuilder {
/**
* Create a term filter for our sorts
*/
- public static TermFilterBuilder sortPropertyTermFilter( final String propertyName ) {
- return FilterBuilders.termFilter( IndexingUtils.FIELD_NAME, propertyName );
+ public static TermQueryBuilder sortPropertyTermFilter( final String propertyName ) {
+ return QueryBuilders.termQuery( IndexingUtils.FIELD_NAME, propertyName );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
index 273f23f..5e2d50f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
@@ -19,19 +19,13 @@
package org.apache.usergrid.persistence.index.query.tree;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.usergrid.persistence.index.exceptions.NoFullTextIndexException;
import org.apache.usergrid.persistence.index.exceptions.NoIndexException;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.impl.GeoSortFields;
-import org.apache.usergrid.persistence.index.query.SortPredicate;
-import org.elasticsearch.index.query.FilterBuilder;
+
import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
import com.google.common.base.Optional;
@@ -108,7 +102,7 @@ public interface QueryVisitor {
* Return any filters created during parsing
* @return
*/
- Optional<FilterBuilder> getFilterBuilder();
+ Optional<QueryBuilder> getFilterBuilder();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
index 3d2517c..5142d76 100644
--- a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
+++ b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
@@ -73,8 +73,6 @@
},
"location": {
"type": "geo_point",
- "lat_lon": true,
- "geohash": true,
"doc_values": true
},
"string": {