You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2016/04/08 22:20:59 UTC
[12/36] usergrid git commit: Unignored and fixed the test that
verified reindexing. It now passes because I've added the reindexing
functionality and proved it works. More tests to be added.
Unignored and fixed the test that verified reindexing. It now passes because I've added the reindexing functionality and proved it works. More tests to be added.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/aebfe1b6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/aebfe1b6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/aebfe1b6
Branch: refs/heads/release-2.1.1
Commit: aebfe1b6db6f4855cb3cff4d371120161106e597
Parents: 2562e97
Author: George Reyes <gr...@apache.org>
Authored: Fri Mar 18 15:44:02 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Thu Mar 24 09:14:50 2016 -0700
----------------------------------------------------------------------
.../rest/applications/CollectionResource.java | 60 ++++++-
.../collection/CollectionsResourceIT.java | 156 ++++++++++++++++---
2 files changed, 187 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/aebfe1b6/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
index b7f5f93..8b05a79 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/applications/CollectionResource.java
@@ -38,7 +38,11 @@ import org.springframework.stereotype.Component;
import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
+import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilderImpl;
+import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
import org.apache.usergrid.rest.RootResource;
@@ -131,6 +135,30 @@ public class CollectionResource extends ServiceResource {
return response;
}
+ @POST
+ @Path("_reindex")
+ @Produces({ MediaType.APPLICATION_JSON,"application/javascript"})
+ @RequireApplicationAccess
+ @JSONP
+ public ApiResponse executePostForReindexing( @Context UriInfo ui, String body,
+ @QueryParam("callback") @DefaultValue("callback") String callback )
+ throws Exception {
+
+ System.out.println();
+// logger.info( "Rebuilding collection {} in application {}", collectionName, applicationIdStr );
+//
+//
+//
+// final UUID appId = UUIDUtils.tryExtractUUID( applicationIdStr );
+//
+ //TODO: check to see how all of this runs with service and query params. Ideally none should be passed in.
+ final ReIndexRequestBuilder request =
+ createRequest().withApplicationId( services.getApplicationId() ).withCollection(
+ String.valueOf( getServiceParameters().get( 0 ) ) );
+//
+ return executeAndCreateResponse( request, callback );
+ }
+
@Override
@Path( RootResource.ENTITY_ID_PATH)
public AbstractContextResource addIdParameter( @Context UriInfo ui, @PathParam("entityId") PathSegment entityId )
@@ -175,9 +203,33 @@ public class CollectionResource extends ServiceResource {
return getSubResource( CollectionResource.class );
}
-// @
-// public ServiceResource doLogicInServiceResource() throws Exception{
-// return getSubResource( ServiceResource.class );
-// }
+ private ReIndexService getReIndexService() {
+ return injector.getInstance( ReIndexService.class );
+ }
+
+ private ReIndexRequestBuilder createRequest() {
+ //TODO: wire this up through spring, and in the future guice.
+ return new ReIndexRequestBuilderImpl();
+ }
+
+ /**
+ * Execute the request and return the response.
+ */
+ private ApiResponse executeAndCreateResponse( final ReIndexRequestBuilder request, final String callback ) {
+
+
+ final ReIndexService.ReIndexStatus status = getReIndexService().rebuildIndex( request );
+
+ final ApiResponse response = createApiResponse();
+
+ response.setAction( "rebuild indexes" );
+ response.setProperty( "jobId", status.getJobId() );
+ response.setProperty( "status", status.getStatus() );
+ response.setProperty( "lastUpdatedEpoch", status.getLastUpdated() );
+ response.setProperty( "numberQueued", status.getNumberProcessed() );
+ response.setSuccess();
+
+ return response;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/aebfe1b6/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index 6d8a5e6..026f8a1 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -123,14 +123,14 @@ public class CollectionsResourceIT extends AbstractRestIT {
*
* @throws Exception
*/
- @Ignore("The reindexing isn't currently supported yet.")
+ @Test
public void postToCollectionSchemaUpdateExistingCollection() throws Exception {
//Create test collection with test entity that is full text indexed.
Entity testEntity = new Entity();
- testEntity.put( "one", "12/31/9999" );
+ testEntity.put( "one", "value" );
//this field shouldn't persist after reindexing.
- testEntity.put( "two","2015-04-20T17:41:38.035Z" );
+ testEntity.put( "two","valuetwo" );
//TODO: add arrays to the indexing test
//testEntity.put("array","array stuff here");
@@ -156,26 +156,39 @@ public class CollectionsResourceIT extends AbstractRestIT {
//Below is what needs to be implemented along with the index call above
//Get the collection schema and verify that it contains the same schema as posted above.
- Collection collection = this.app().collection( "testCollection" ).collection( "_index" ).get();
-
- LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData();
- //TODO: the below will have to be replaced by the values that I deem correct.
- assertEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated" ));
- assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy" ) );
- assertEquals( ( thing ).get( "lastReindexed" ),testCollectionSchema.get( "lastReindexed" ) );
-
- //TODO: this test doesn't check to see if create checks the schema. Only that the reindex removes whats already there.
- ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" );
- assertEquals( "one",schema.get( 0 ) );
+// Collection collection = this.app().collection( "testCollection" ).collection( "_index" ).get();
+//
+// LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData();
+// //TODO: the below will have to be replaced by the values that I deem correct.
+// assertEquals( ( thing ).get( "lastUpdated" ), testCollectionSchema.get( "lastUpdated" ));
+// assertEquals( ( thing ).get( "lastUpdateBy" ),testCollectionSchema.get( "lastUpdateBy" ) );
+// assertEquals( ( thing ).get( "lastReindexed" ),testCollectionSchema.get( "lastReindexed" ) );
+//
+// //TODO: this test doesn't check to see if create checks the schema. Only that the reindex removes whats already there.
+// ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" );
+// assertEquals( "one",schema.get( 0 ) );
//Reindex and verify that the entity only has field one index.
this.app().collection( "testCollection" ).collection( "_reindex" ).post();
- refreshIndex();
+ String query = "one ='value'";
+ QueryParameters queryParameters = new QueryParameters().setQuery(query);
- Entity reindexedEntity = this.app().collection( "testCollection" ).entity( returnedEntity.getUuid() ).get();
- assertEquals( "12/31/9999",reindexedEntity.get( "one" ) );
- assertNull( reindexedEntity.get( "two" ) );
+ //having a name breaks it. Need to get rid of the stack trace and also
+ Collection tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
+ Entity reindexedEntity = tempEntity.getResponse().getEntity();
+ assertEquals( "value",reindexedEntity.get( "one" ) );
+
+ //Verify if you can query on an entity that was not indexed and that no entities are returned.
+ query = "two = 'valuetwo'";
+ queryParameters = new QueryParameters().setQuery(query);
+ tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
+ assertEquals(0,tempEntity.getResponse().getEntities().size());
+
+// refreshIndex();
+//
+// Entity reindexedEntity = this.app().collection( "testCollection" ).entity( returnedEntity.getUuid() ).get();
+// assertEquals( "12/31/9999",reindexedEntity.get( "one" ) );
//not sure if this should have some kind of sleep here because this reindex will be heavily throttled.
}
@@ -318,12 +331,6 @@ public class CollectionsResourceIT extends AbstractRestIT {
Entity reindexedEntity = tempEntity.getResponse().getEntity();
assertEquals( "value2",((Map)reindexedEntity.get( "one" )).get( "anotherKey" ) );
- //Verify if you can query on an entity that was not indexed and that no entities are returned.
- //TODO: check that the below gets indexed as well. although the above should prove that at least one thing is getting indexed.
-// query = "one.anotherKey = 'value2'";
-// queryParameters = new QueryParameters().setQuery(query);
-// tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
-// assertEquals(0,tempEntity.getResponse().getEntities().size());
}
@@ -373,7 +380,6 @@ public class CollectionsResourceIT extends AbstractRestIT {
assertEquals( "value2",((Map)reindexedEntity.get( "one" )).get( "anotherKey" ) );
//Verify if you can query on an entity that was not indexed and that no entities are returned.
- //TODO: check that the below gets indexed as well. although the above should prove that at least one thing is getting indexed.
query = "one.anotherKey = 'value2'";
queryParameters = new QueryParameters().setQuery(query);
tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
@@ -382,6 +388,106 @@ public class CollectionsResourceIT extends AbstractRestIT {
}
+ @Test
+ public void postToCollectionSchemaArrayWithSelectiveTopLevelIndexingAddingDefaultPropertyNames() throws Exception {
+
+ //Include the property labeled two to be index.
+ ArrayList<String> indexingArray = new ArrayList<>( );
+ indexingArray.add( "two" );
+ //this should work such that one.key and one.anotherKey should work.
+ indexingArray.add( "one.key" );
+ indexingArray.add( "name" );
+
+ //field "fields" is required.
+ Entity payload = new Entity();
+ payload.put( "fields", indexingArray);
+
+ //Post index to the collection metadata
+ this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload );
+ refreshIndex();
+
+ Map<String,Object> arrayFieldsForTestingSelectiveIndexing = new HashMap<>();
+
+ arrayFieldsForTestingSelectiveIndexing.put( "wowMoreKeys","value" );
+ arrayFieldsForTestingSelectiveIndexing.put( "thisShouldBeQueryableToo","value2");
+
+ Map<String,Object> arrayFieldsForTesting = new HashMap<>();
+
+ arrayFieldsForTesting.put( "key",arrayFieldsForTestingSelectiveIndexing );
+ arrayFieldsForTesting.put( "anotherKey","value2");
+
+ //Create test collection with a test entity that is partially indexed.
+ Entity testEntity = new Entity();
+ testEntity.put( "one", arrayFieldsForTesting );
+ testEntity.put( "two","query" );
+ testEntity.put( "name","howdy");
+
+ //Post entity.
+ this.app().collection( "testCollection" ).post( testEntity );
+ refreshIndex();
+
+ //Do a query to see if you can find the indexed query.
+ String query = "name = 'howdy'";
+ QueryParameters queryParameters = new QueryParameters().setQuery(query);
+
+ //having a name breaks it. Need to get rid of the stack trace and also
+ Collection tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
+ Entity reindexedEntity = tempEntity.getResponse().getEntity();
+ assertEquals( "howdy",(reindexedEntity.get( "name" )) );
+
+ }
+
+
+ /**
+ * Verify that doing a put goes through and still applies the existing schema.
+ * @throws Exception
+ */
+ @Test
+ public void putToCollectionSchema() throws Exception {
+
+ //Include the property labeled two to be index.
+ ArrayList<String> indexingArray = new ArrayList<>( );
+ indexingArray.add( "one" );
+
+ //field "fields" is required.
+ Entity payload = new Entity();
+ payload.put( "fields", indexingArray);
+
+ //Post index to the collection metadata
+ this.app().collection( "testCollection" ).collection( "_indexes" ).post( payload );
+ refreshIndex();
+
+ //Create test collection with a test entity that is partially indexed.
+ Entity testEntity = new Entity();
+ testEntity.put( "one", "two");
+ testEntity.put( "test", "anotherTest");
+
+ //Post entity.
+ Entity postedEntity = this.app().collection( "testCollection" ).post( testEntity );
+ refreshIndex();
+
+ testEntity.put( "one","three" );
+ this.app().collection( "testCollection" ).entity( postedEntity.getUuid() ).put( testEntity );
+ refreshIndex();
+
+ //Do a query to see if you can find the indexed query.
+ String query = "one = 'three'";
+ QueryParameters queryParameters = new QueryParameters().setQuery(query);
+
+ //having a name breaks it. Need to get rid of the stack trace and also
+ Collection tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
+ Entity reindexedEntity = tempEntity.getResponse().getEntity();
+ assertEquals( "three",(reindexedEntity.get( "one" )) );
+
+ //check that the old value no longer exists in elasticsearch
+ query = "one = 'two'";
+ queryParameters = new QueryParameters().setQuery(query);
+ tempEntity = this.app().collection( "testCollection" ).get(queryParameters,true);
+ assertEquals(0,tempEntity.getResponse().getEntities().size());
+
+ }
+
+
/**