You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2016/04/19 18:51:31 UTC
[34/50] usergrid git commit: Disable indexing per Entity type for
USERGRID-1278. https://issues.apache.org/jira/browse/USERGRID-1278
Disable indexing per Entity type for USERGRID-1278. https://issues.apache.org/jira/browse/USERGRID-1278
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fbced588
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fbced588
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fbced588
Branch: refs/heads/master
Commit: fbced588bdf7de5f7192d20e2da6a8740df0218a
Parents: 2e576f3
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Apr 14 12:26:06 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Apr 14 12:26:06 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 86 ++++++++++++++------
.../collection/CollectionsResourceIT.java | 56 +++++++++++--
2 files changed, 112 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbced588/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 891c1fe..b2330f3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -601,14 +601,36 @@ public class CpEntityManager implements EntityManager {
handleWriteUniqueVerifyException( entity, wuve );
}
- // queue an event to update the new entity
- indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+ if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
+ // queue an event to update the new entity
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
- // queue up an event to clean-up older versions than this one from the index
- if(entityManagerFig.getDeindexOnUpdate()) {
- indexService.queueDeIndexOldVersion(applicationScope, entityId);
+ // queue up an event to clean-up older versions than this one from the index
+ if (entityManagerFig.getDeindexOnUpdate()) {
+ indexService.queueDeIndexOldVersion( applicationScope, entityId );
+ }
+ }
+ }
+
+ private boolean skipIndexingForType( String type ) {
+
+ boolean skipIndexing = false;
+
+ MapManager mm = getMapManagerForTypes();
+ IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm );
+ String collectionName = Schema.defaultCollectionName( type );
+ Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName );
+
+ if ( collectionIndexingSchema.isPresent()) {
+ Map jsonMapData = collectionIndexingSchema.get();
+ final ArrayList fields = (ArrayList) jsonMapData.get( "fields" );
+ if ( fields.size() == 1 && fields.get(0).equals("!")) {
+ skipIndexing = true;
+ }
}
+
+ return skipIndexing;
}
@@ -670,8 +692,9 @@ public class CpEntityManager implements EntityManager {
Id entityId = new SimpleId( entityRef.getUuid(), entityRef.getType() );
- //Step 4 && 5
- indexService.queueEntityDelete( applicationScope, entityId );
+ if ( !skipIndexingForType( entityId.getType() ) ) {
+ indexService.queueEntityDelete( applicationScope, entityId );
+ }
//Step 6
//delete from our UUID index
@@ -714,7 +737,8 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public Results searchCollectionConsistent( EntityRef entityRef, String collectionName, Query query, int expectedResults) throws Exception {
+ public Results searchCollectionConsistent(
+ EntityRef entityRef, String collectionName, Query query, int expectedResults) throws Exception {
return getRelationManager( entityRef ).searchCollectionConsistent(collectionName, query, expectedResults);
}
@@ -754,8 +778,8 @@ public class CpEntityManager implements EntityManager {
public RelationManager getRelationManager( EntityRef entityRef ) {
Preconditions.checkNotNull(entityRef, "entityRef cannot be null");
- CpRelationManager relationManager =
- new CpRelationManager(managerCache, indexService, collectionService, connectionService, this, entityManagerFig, applicationId, entityRef );
+ CpRelationManager relationManager = new CpRelationManager( managerCache, indexService, collectionService,
+ connectionService, this, entityManagerFig, applicationId, entityRef );
return relationManager;
}
@@ -830,7 +854,8 @@ public class CpEntityManager implements EntityManager {
Timer.Context repairedEntityGet = entGetRepairedEntityTimer.time();
- //TODO: can't we just sub in the getEntityRepair method here so for every read of a uniqueEntityField we can verify it is correct?
+ // TODO: can't we just sub in the getEntityRepair method here
+ // so for every read of a uniqueEntityField we can verify it is correct?
StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString());
@@ -916,8 +941,8 @@ public class CpEntityManager implements EntityManager {
// add a warn statement so we can see if we have data migration issues.
// TODO When we get an event system, trigger a repair if this is detected
if ( results.size() > 1 ) {
- logger.warn( "More than 1 entity with Owner id '{}' of type '{}' and alias '{}' exists. This is a duplicate alias, and needs audited",
- ownerRef, collectionType, aliasValue );
+ logger.warn( "More than 1 entity with Owner id '{}' of type '{}' and alias '{}' exists. " +
+ "This is a duplicate alias, and needs audited", ownerRef, collectionType, aliasValue );
}
return results.get(aliasValue);
@@ -1139,7 +1164,9 @@ public class CpEntityManager implements EntityManager {
//Adding graphite metrics
- indexService.queueEntityIndexUpdate(applicationScope, cpEntity, 0);
+ if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
+ indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+ }
}
@@ -1785,12 +1812,17 @@ public class CpEntityManager implements EntityManager {
//do a check to see if you have a * field. If you do have a * field then ignore all other fields
//and only accept the * field.
- if(fieldProperties.contains( "*" )){
- ArrayList<String> wildCardArrayList = new ArrayList<>( );
+ if ( fieldProperties.contains( "*" )) {
+ ArrayList<String> wildCardArrayList = new ArrayList<>();
wildCardArrayList.add( "*" );
- schemaMap.put( "fields",wildCardArrayList );
- }
- else {
+ schemaMap.put( "fields", wildCardArrayList );
+
+ } else if ( fieldProperties.contains( "!" )) {
+ ArrayList<String> wildCardArrayList = new ArrayList<>();
+ wildCardArrayList.add( "!" );
+ schemaMap.put( "fields", wildCardArrayList );
+
+ } else {
schemaMap.putAll( properties );
}
@@ -1815,7 +1847,7 @@ public class CpEntityManager implements EntityManager {
public Object getCollectionSchema( String collectionName ){
MapManager mm = getMapManagerForTypes();
- IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm ); //managerCache.getIndexSchema( mm );
+ IndexSchemaCache indexSchemaCache = indexSchemaCacheFactory.getInstance( mm );
Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName );
@@ -2388,7 +2420,8 @@ public class CpEntityManager implements EntityManager {
if ( !skipAggregateCounters ) {
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> m = createMutator( cass.getApplicationKeyspace( applicationId ), be );
- counterUtils.batchIncrementAggregateCounters( m, applicationId, userId, groupId, null, category, counters, timestamp );
+ counterUtils.batchIncrementAggregateCounters(
+ m, applicationId, userId, groupId, null, category, counters, timestamp );
//Adding graphite metrics
Timer.Context timeIncrementCounters =entIncrementAggregateCountersTimer.time();
@@ -2987,8 +3020,9 @@ public class CpEntityManager implements EntityManager {
final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId() );
- return graphManager.getEdgeTypesFromSource( searchByEdgeType ).map( edgeName -> getConnectionNameFromEdgeName( edgeName ) ).collect(
- () -> new HashSet<String>(), ( r, s ) -> r.add( s ) ).toBlocking().last();
+ return graphManager.getEdgeTypesFromSource(
+ searchByEdgeType ).map( edgeName -> getConnectionNameFromEdgeName( edgeName ) )
+ .collect( () -> new HashSet<String>(), ( r, s ) -> r.add( s ) ).toBlocking().last();
}
@@ -3001,7 +3035,8 @@ public class CpEntityManager implements EntityManager {
final SearchEdgeType searchByEdgeType = createConnectionTypeSearch( entityRef.asId() );
return graphManager.getEdgeTypesToTarget(searchByEdgeType).map(
- edgeName -> getConnectionNameFromEdgeName( edgeName ) ).collect( () -> new HashSet<String>( ), ( r, s ) -> r.add(s) ).toBlocking().last();
+ edgeName -> getConnectionNameFromEdgeName( edgeName ) )
+ .collect( () -> new HashSet<String>( ), ( r, s ) -> r.add(s) ).toBlocking().last();
}
@@ -3030,7 +3065,8 @@ public class CpEntityManager implements EntityManager {
try {
for (int i = 0; i < 20; i++) {
if (searchCollection(
- new SimpleEntityRef(org.apache.usergrid.persistence.entities.Application.ENTITY_TYPE, getApplicationId()),
+ new SimpleEntityRef(
+ org.apache.usergrid.persistence.entities.Application.ENTITY_TYPE, getApplicationId()),
InflectionUtils.pluralize("refresh"),
Query.fromQL("select * where uuid='" + refreshEntity.getUuid() + "'")
).size() > 0
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fbced588/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
index d052564..b240629 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/collection/CollectionsResourceIT.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.rest.applications.collection;
import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.entities.Application;
@@ -42,11 +43,7 @@ import javax.ws.rs.client.Invocation;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.ArrayList;
+import java.util.*;
import org.apache.commons.lang.NullArgumentException;
@@ -943,4 +940,53 @@ public class CollectionsResourceIT extends AbstractRestIT {
Assert.assertEquals( "Should not be more than one name property", -1, secondFred );
*/
}
+
+
+ /**
+ * Test that when schema is "!" entity gets saved but does not get indexed
+ */
+ @Test
+ public void postCollectionSchemaWithWildcardIndexNone() throws Exception {
+
+ // creating schema with no index wildcard and other fields that should be ignored
+ ArrayList<String> indexingArray = new ArrayList<>( );
+ indexingArray.add( "!" );
+ indexingArray.add( "one" );
+ indexingArray.add( "two" );
+ Entity payload = new Entity();
+ payload.put( "fields", indexingArray);
+
+ String randomizer = RandomStringUtils.randomAlphanumeric(10);
+ String collectionName = "col_" + randomizer;
+ app().collection( collectionName ).collection( "_indexes" ).post( payload );
+ refreshIndex();
+
+ // was the no-index wildcard saved and others ignored?
+ Collection collection = app().collection( collectionName ).collection( "_index" ).get();
+ LinkedHashMap testCollectionSchema = (LinkedHashMap)collection.getResponse().getData();
+ ArrayList<String> schema = ( ArrayList<String> ) testCollectionSchema.get( "fields" );
+ assertTrue( schema.contains( "!" ) );
+ assertFalse( schema.contains( "one" ) );
+ assertFalse( schema.contains( "two" ) );
+
+ // post an entity with a name and a color
+ String entityName = "name_" + randomizer;
+ Entity postedEntity = this.app().collection( collectionName )
+ .post( new Entity().withProp("name", entityName).withProp( "color", "magenta" ) );
+
+ // should be able to get entity by ID
+ Entity getByIdEntity = app().collection( collectionName ).entity( postedEntity.getUuid() ).get();
+ assertNotNull( getByIdEntity );
+ assertEquals( postedEntity.getUuid(), getByIdEntity.getUuid() );
+
+ // should be able to get entity by name
+ Entity getByNameEntity = app().collection( collectionName ).entity( entityName ).get();
+ assertNotNull( getByNameEntity );
+ assertEquals( postedEntity.getUuid(), getByNameEntity.getUuid() );
+
+ // should NOT be able to get entity by query
+ Iterator<Entity> getByQuery = app().collection( collectionName )
+ .get( new QueryParameters().setQuery( "select * where color='magenta'" ) ).iterator();
+ assertFalse( getByQuery.hasNext() );
+ }
}