You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2016/04/08 22:21:21 UTC
[34/36] usergrid git commit: Add some more comments and clarity to
selective indexing.
Add some more comments and clarity to selective indexing.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9f3720e9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9f3720e9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9f3720e9
Branch: refs/heads/release-2.1.1
Commit: 9f3720e998edd6343d78ec1b5360f1ce244a4d49
Parents: e5ae2a5
Author: George Reyes <gr...@apache.org>
Authored: Mon Apr 4 10:55:44 2016 -0700
Committer: George Reyes <gr...@apache.org>
Committed: Mon Apr 4 10:55:44 2016 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 36 ++++++++------------
.../index/ReIndexServiceImpl.java | 1 -
.../core/consistency/TimeServiceImpl.java | 5 ++-
.../index/impl/EntityToMapConverter.java | 23 +++++++------
4 files changed, 31 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 6bbf0a3..4ee497c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -18,14 +18,11 @@ package org.apache.usergrid.corepersistence;
import java.nio.ByteBuffer;
import java.time.Instant;
-import java.time.ZoneOffset;
-import java.time.ZonedDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -38,22 +35,15 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
-
-import org.apache.usergrid.corepersistence.index.IndexSchemaCache;
-import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
-import org.apache.usergrid.corepersistence.service.CollectionService;
-import org.apache.usergrid.corepersistence.service.ConnectionService;
-import org.apache.usergrid.persistence.index.EntityIndex;
-import org.apache.usergrid.utils.*;
-import org.apache.usergrid.utils.ClassUtils;
-import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
-import org.apache.avro.generic.GenericData;
-
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
+import org.apache.usergrid.corepersistence.index.IndexSchemaCache;
+import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
+import org.apache.usergrid.corepersistence.service.CollectionService;
+import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
@@ -97,6 +87,7 @@ import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
import org.apache.usergrid.persistence.graph.GraphManager;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.SearchEdgeType;
+import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.map.MapManager;
@@ -106,6 +97,13 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import org.apache.usergrid.utils.ClassUtils;
+import org.apache.usergrid.utils.CompositeUtils;
+import org.apache.usergrid.utils.InflectionUtils;
+import org.apache.usergrid.utils.Inflector;
+import org.apache.usergrid.utils.JsonUtils;
+import org.apache.usergrid.utils.StringUtils;
+import org.apache.usergrid.utils.UUIDUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
@@ -1751,17 +1749,14 @@ public class CpEntityManager implements EntityManager {
@Override
public Map createCollectionSchema( String collectionName, String owner ,Map<String, Object> properties ){
- //haven't decided which one I should base off of which, maybe from epoch to utc
//TODO: change timeservice as below then use timeservice.
- //TODO: only allow a single reindex in elasticsearch at a time. akka.
Instant timeInstance = Instant.now();
Long epoch = timeInstance.toEpochMilli();
Map<String,Object> schemaMap = new HashMap<>( );
-
schemaMap.put("lastUpdated",epoch);
//this needs the method that can extract the user from the token no matter the token.
//Possible values are app credentials, org credentials, or the user email(Admin tokens).
@@ -1775,7 +1770,7 @@ public class CpEntityManager implements EntityManager {
java.util.Optional<Map> collectionIndexingSchema = indexSchemaCache.getCollectionSchema( collectionName );
- //If we do have a schema then parse it and add it to a list of properties we want to keep.Otherwise return.
+ //If there is an existing schema then take the lastReindexed time and keep it around.Otherwise initialize to 0.
if ( collectionIndexingSchema.isPresent() ) {
Map jsonMapData = collectionIndexingSchema.get();
schemaMap.put( "lastReindexed", jsonMapData.get( "lastReindexed" ) );
@@ -1786,6 +1781,8 @@ public class CpEntityManager implements EntityManager {
ArrayList<String> fieldProperties = ( ArrayList<String> ) properties.get( "fields" );
+ //do a check to see if you have a * field. If you do have a * field then ignore all other fields
+ //and only accept the * field.
if(fieldProperties.contains( "*" )){
ArrayList<String> wildCardArrayList = new ArrayList<>( );
wildCardArrayList.add( "*" );
@@ -1795,9 +1792,6 @@ public class CpEntityManager implements EntityManager {
schemaMap.putAll( properties );
}
- //do a check to see if you have a * field. If you do have a * field then ignore all other fields
- //and only accept the * field. That logic goes below and in the put.
-
indexSchemaCache.putCollectionSchema( collectionName, JsonUtils.mapToJsonString( schemaMap ) );
return schemaMap;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index 125222a..0aa0f60 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -44,7 +44,6 @@ import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapManagerFactory;
import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.InflectionUtils;
import org.apache.usergrid.utils.JsonUtils;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java
index 00a58d3..99fe756 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/consistency/TimeServiceImpl.java
@@ -20,6 +20,9 @@
package org.apache.usergrid.persistence.core.consistency;
+import java.time.Instant;
+
+
/**
* Simple time service to get the current system time.
*/
@@ -28,6 +31,6 @@ public class TimeServiceImpl implements TimeService{
@Override
public long getCurrentTime() {
- return System.currentTimeMillis();
+ return Instant.now().toEpochMilli();
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9f3720e9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
index 98a8ed5..8eb9f48 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EntityToMapConverter.java
@@ -65,6 +65,7 @@ public class EntityToMapConverter {
* @param applicationScope
* @param entity The entity
* @param indexEdge The edge this entity is indexed on
+ * @param fieldsToIndex A set of fields that will be indexed should they exist on the entity. Other fields will be filtered out.
*/
public static Map<String, Object> convert(ApplicationScope applicationScope, final IndexEdge indexEdge,
final Entity entity, Optional<Set<String>> fieldsToIndex) {
@@ -114,7 +115,6 @@ public class EntityToMapConverter {
if(fieldsToIndex.isPresent()){
Set<String> defaultProperties = fieldsToIndex.get();
- //copy paste logic here.
HashSet mapFields = ( HashSet ) outputEntity.get( "fields" );
Iterator collectionIterator = mapFields.iterator();
@@ -137,10 +137,11 @@ public class EntityToMapConverter {
}
/**
- * This method is crucial for selective top level indexing. Here we check to see if the flatted properties
- * are in fact a top level exclusion e.g one.two.three and one.three.two can be allowed for querying by
- * specifying one in the schema. If they are not a top level exclusion then they are removed from the iterator and
- * the map.
+ * Handles checking to see if a field is a top level exclusion or just a field that shouldn't be indexed.
+ * This is handled by looping through all the fields we want to be able to query on, and checking to see if a
+ * specific field name is included. If the field name is included then do nothing. If the field name is not included
+ * then we do not want to be able to query on it. Instead we remove it from the collectionIterator which
+ * removes it from the outputEntity above, thus filtering it out.
*
* @param fieldsToKeep - contains a list of fields that the user defined in their schema.
* @param collectionIterator - contains the iterator with the reference to the map where we want to remove the field. Once removed here it is removed from the entity so it won't be indexed.
@@ -154,12 +155,12 @@ public class EntityToMapConverter {
- //goes through a loop of all the fields ( excluding default ) that we want to keep.
- //if the toRemoveFlag is set to false then we want to keep the property, otherwise we set it to true and remove
+ //goes through a loop of all the fields that we want to keep.
+ //if the toRemoveFlag is set to false then we want to keep the property and do nothing to it, otherwise we set it to true and remove
//the property.
while ( fieldIterator.hasNext() ) {
//this is the field that we're
- String requiredInclusionString = ( String ) fieldIterator.next();
+ String fieldToKeep = ( String ) fieldIterator.next();
//Since we know that the fieldName cannot be equal to the requiredInclusion criteria due to the if condition before we enter this method
@@ -169,9 +170,9 @@ public class EntityToMapConverter {
//The second part of the if loop also requires that the fieldName is followed by a period after we check to ensure that the
//indexing criteria is included in the string. This is done to weed out values such as one.twoexample.three
// when we should only keep one.two.three when comparing the indexing criteria of one.two.
- if(fieldName.length() > requiredInclusionString.length()
- && fieldName.contains( requiredInclusionString )
- && fieldName.charAt( requiredInclusionString.length() )=='.' ) {
+ if(fieldName.length() > fieldToKeep.length()
+ && fieldName.contains( fieldToKeep )
+ && fieldName.charAt( fieldToKeep.length() )=='.' ) {
toRemoveFlag = false;
break;
}